# -*- coding: utf-8 -*-
"""The :class:`Schema` class, including its metaclass and options (class Meta)."""
from __future__ import absolute_import, unicode_literals

from collections import defaultdict, Mapping, OrderedDict
import functools
import copy
import datetime as dt
import decimal
import inspect
import json
import uuid
import warnings

from marshmallow import base, fields, utils, class_registry, marshalling
from marshmallow.compat import (
    binary_type,
    iteritems,
    iterkeys,
    text_type,
    with_metaclass,
)
from marshmallow.exceptions import ValidationError, StringNotCollectionError
from marshmallow.orderedset import OrderedSet
from marshmallow.decorators import (
    POST_DUMP,
    POST_LOAD,
    PRE_DUMP,
    PRE_LOAD,
    VALIDATES,
    VALIDATES_SCHEMA,
)
from marshmallow.utils import RAISE, missing, is_collection

def _get_fields(attrs, field_class, pop=False, ordered=False):
    """Get fields from a class. If ordered=True, fields will sorted by creation index.

    :param attrs: Mapping of class attributes
    :param type field_class: Base field class
    :param bool pop: Remove matching fields
    """
    fields = [
        (field_name, field_value)
        for field_name, field_value in iteritems(attrs)
        if utils.is_instance_or_subclass(field_value, field_class)
    ]
    if pop:
        for field_name, _ in fields:
            del attrs[field_name]
    if ordered:
        fields.sort(key=lambda pair: pair[1]._creation_index)
    return fields

# This function allows Schemas to inherit from non-Schema classes and ensures
#   inheritance according to the MRO
def _get_fields_by_mro(klass, field_class, ordered=False):
    """Collect fields from a class, following its method resolution order. The
    class itself is excluded from the search; only its parents are checked. Get
    fields from ``_declared_fields`` if available, else use ``__dict__``.

    :param type klass: Class whose fields to retrieve
    :param type field_class: Base field class
    """
    mro = inspect.getmro(klass)
    # Loop over mro in reverse to maintain correct order of fields
    return sum(
        (
            _get_fields(
                getattr(base, '_declared_fields', base.__dict__),
                field_class,
                ordered=ordered,
            )
            for base in mro[:0:-1]
        ),
        [],
    )

class SchemaMeta(type):
    """Metaclass for the Schema class. Binds the declared fields to
    a ``_declared_fields`` attribute, which is a dictionary mapping attribute
    names to field objects. Also sets the ``opts`` class attribute, which is
    the Schema class's ``class Meta`` options.
    """

    def __new__(mcs, name, bases, attrs):
        meta = attrs.get('Meta')
        ordered = getattr(meta, 'ordered', False)
        if not ordered:
            # Inherit 'ordered' option
            # Warning: We loop through bases instead of MRO because we don't
            # yet have access to the class object
            # (i.e. can't call super before we have fields)
            for base_ in bases:
                if hasattr(base_, 'Meta') and hasattr(base_.Meta, 'ordered'):
                    ordered = base_.Meta.ordered
                    break
            else:
                ordered = False
        cls_fields = _get_fields(attrs, base.FieldABC, pop=True, ordered=ordered)
        klass = super(SchemaMeta, mcs).__new__(mcs, name, bases, attrs)
        inherited_fields = _get_fields_by_mro(klass, base.FieldABC, ordered=ordered)

        # Use getattr rather than attrs['Meta'] so that we get inheritance for free
        meta = getattr(klass, 'Meta')
        # Set klass.opts in __new__ rather than __init__ so that it is accessible in
        # get_declared_fields
        klass.opts = klass.OPTIONS_CLASS(meta, ordered=ordered)
        # Add fields specifid in the `include` class Meta option
        cls_fields += list(klass.opts.include.items())

        dict_cls = OrderedDict if ordered else dict
        # Assign _declared_fields on class
        klass._declared_fields = mcs.get_declared_fields(
            klass=klass,
            cls_fields=cls_fields,
            inherited_fields=inherited_fields,
            dict_cls=dict_cls,
        )
        return klass

    @classmethod
    def get_declared_fields(mcs, klass, cls_fields, inherited_fields, dict_cls):
        """Returns a dictionary of field_name => `Field` pairs declard on the class.
        This is exposed mainly so that plugins can add additional fields, e.g. fields
        computed from class Meta options.

        :param type klass: The class object.
        :param dict cls_fields: The fields declared on the class, including those added
            by the ``include`` class Meta option.
        :param dict inherited_fileds: Inherited fields.
        :param type dict_class: Either `dict` or `OrderedDict`, depending on the whether
            the user specified `ordered=True`.
        """
        return dict_cls(inherited_fields + cls_fields)

    # NOTE: self is the class object
    def __init__(self, name, bases, attrs):
        super(SchemaMeta, self).__init__(name, bases, attrs)
        if name:
            class_registry.register(name, self)
        self._hooks = self.resolve_hooks()

    def resolve_hooks(self):
        """Add in the decorated processors

        By doing this after constructing the class, we let standard inheritance
        do all the hard work.
        """
        mro = inspect.getmro(self)

        hooks = defaultdict(list)

        for attr_name in dir(self):
            # Need to look up the actual descriptor, not whatever might be
            # bound to the class. This needs to come from the __dict__ of the
            # declaring class.
            for parent in mro:
                try:
                    attr = parent.__dict__[attr_name]
                except KeyError:
                    continue
                else:
                    break
            else:
                # In case we didn't find the attribute and didn't break above.
                # We should never hit this - it's just here for completeness
                # to exclude the possibility of attr being undefined.
                continue

            try:
                hook_config = attr.__marshmallow_hook__
            except AttributeError:
                pass
            else:
                for key in iterkeys(hook_config):
                    # Use name here so we can get the bound method later, in
                    # case the processor was a descriptor or something.
                    hooks[key].append(attr_name)

        return hooks

class SchemaOpts(object):
    """class Meta options for the :class:`Schema`. Defines defaults."""

    def __init__(self, meta, ordered=False):
        self.fields = getattr(meta, 'fields', ())
        if not isinstance(self.fields, (list, tuple)):
            raise ValueError('`fields` option must be a list or tuple.')
        self.additional = getattr(meta, 'additional', ())
        if not isinstance(self.additional, (list, tuple)):
            raise ValueError('`additional` option must be a list or tuple.')
        if self.fields and self.additional:
            raise ValueError(
                'Cannot set both `fields` and `additional` options'
                ' for the same Schema.',
            )
        self.exclude = getattr(meta, 'exclude', ())
        if not isinstance(self.exclude, (list, tuple)):
            raise ValueError('`exclude` must be a list or tuple.')
        self.dateformat = getattr(meta, 'dateformat', None)
        if hasattr(meta, 'json_module'):
            warnings.warn(
                'The json_module class Meta option is deprecated. Use render_module instead.',
                DeprecationWarning,
            )
            render_module = getattr(meta, 'json_module', json)
        else:
            render_module = json
        self.render_module = getattr(meta, 'render_module', render_module)
        self.ordered = getattr(meta, 'ordered', ordered)
        self.index_errors = getattr(meta, 'index_errors', True)
        self.include = getattr(meta, 'include', {})
        self.load_only = getattr(meta, 'load_only', ())
        self.dump_only = getattr(meta, 'dump_only', ())
        self.unknown = getattr(meta, 'unknown', RAISE)

class BaseSchema(base.SchemaABC):
    """Base schema class with which to define custom schemas.

    Example usage:

    .. code-block:: python

        import datetime as dt
        from marshmallow import Schema, fields

        class Album(object):
            def __init__(self, title, release_date):
                self.title = title
                self.release_date = release_date

        class AlbumSchema(Schema):
            title = fields.Str()
            release_date = fields.Date()

        # Or, equivalently
        class AlbumSchema2(Schema):
            class Meta:
                fields = ("title", "release_date")

        album = Album("Beggars Banquet", dt.date(1968, 12, 6))
        schema = AlbumSchema()
        data, errors = schema.dump(album)
        data  # {'release_date': '1968-12-06', 'title': 'Beggars Banquet'}

    :param tuple|list only: Whitelist of the declared fields to select when
        instantiating the Schema. If None, all fields are used. Nested fields
        can be represented with dot delimiters.
    :param tuple|list exclude: Blacklist of the declared fields to exclude
        when instantiating the Schema. If a field appears in both `only` and
        `exclude`, it is not used. Nested fields can be represented with dot
        delimiters.
    :param str prefix: Optional prefix that will be prepended to all the
        serialized field names.
    :param bool many: Should be set to `True` if ``obj`` is a collection
        so that the object will be serialized to a list.
    :param dict context: Optional context passed to :class:`fields.Method` and
        :class:`fields.Function` fields.
    :param tuple|list load_only: Fields to skip during serialization (write-only fields)
    :param tuple|list dump_only: Fields to skip during deserialization (read-only fields)
    :param bool|tuple partial: Whether to ignore missing fields. If its value
        is an iterable, only missing fields listed in that iterable will be
        ignored.
    :param unknown: Whether to exclude, include, or raise an error for unknown
        fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.

    .. versionchanged:: 2.0.0
        `__validators__`, `__preprocessors__`, and `__data_handlers__` are removed in favor of
        `marshmallow.decorators.validates_schema`,
        `marshmallow.decorators.pre_load` and `marshmallow.decorators.post_dump`.
        `__accessor__` and `__error_handler__` are deprecated. Implement the
        `handle_error` and `get_attribute` methods instead.
        """
    TYPE_MAPPING = {
        text_type: fields.String,
        binary_type: fields.String,
        dt.datetime: fields.DateTime,
        float: fields.Float,
        bool: fields.Boolean,
        tuple: fields.Raw,
        list: fields.Raw,
        set: fields.Raw,
        int: fields.Integer,
        uuid.UUID: fields.UUID,
        dt.time: fields.Time,
        dt.date: fields.Date,
        dt.timedelta: fields.TimeDelta,
        decimal.Decimal: fields.Decimal,
    }

    OPTIONS_CLASS = SchemaOpts

    class Meta(object):
        """Options object for a Schema.

        Example usage: ::

            class Meta:
                fields = ("id", "email", "date_created")
                exclude = ("password", "secret_attribute")

        Available options:

        - ``fields``: Tuple or list of fields to include in the serialized result.
        - ``additional``: Tuple or list of fields to include *in addition* to the
            explicitly declared fields. ``additional`` and ``fields`` are
            mutually-exclusive options.
        - ``include``: Dictionary of additional fields to include in the schema. It is
            usually better to define fields as class variables, but you may need to
            use this option, e.g., if your fields are Python keywords. May be an
            `OrderedDict`.
        - ``exclude``: Tuple or list of fields to exclude in the serialized result.
            Nested fields can be represented with dot delimiters.
        - ``dateformat``: Date format for all DateTime fields that do not have their
            date format explicitly specified.
        - ``render_module``: Module to use for `loads` and `dumps`. Defaults to
            `json` from the standard library.
            Defaults to the ``json`` module in the stdlib.
        - ``ordered``: If `True`, order serialization output according to the
            order in which fields were declared. Output of `Schema.dump` will be a
            `collections.OrderedDict`.
        - ``index_errors``: If `True`, errors dictionaries will include the index
            of invalid items in a collection.
        - ``load_only``: Tuple or list of fields to exclude from serialized results.
        - ``dump_only``: Tuple or list of fields to exclude from deserialization
        - ``unknown``: Whether to exclude, include, or raise an error for unknown
            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.
        """
        pass

    def __init__(
        self, only=None, exclude=(), prefix='', many=False,
        context=None, load_only=(), dump_only=(), partial=False,
        unknown=None,
    ):
        # Raise error if only or exclude is passed as string, not list of strings
        if only is not None and not is_collection(only):
            raise StringNotCollectionError('"only" should be a list of strings')
        if exclude is not None and not is_collection(exclude):
            raise StringNotCollectionError('"exclude" should be a list of strings')
        # copy declared fields from metaclass
        self.declared_fields = copy.deepcopy(self._declared_fields)
        self.many = many
        self.only = only
        self.exclude = exclude
        self.prefix = prefix
        self.ordered = self.opts.ordered
        self.load_only = set(load_only) or set(self.opts.load_only)
        self.dump_only = set(dump_only) or set(self.opts.dump_only)
        self.partial = partial
        self.unknown = unknown or self.opts.unknown
        #: Dictionary mapping field_names -> :class:`Field` objects
        self.fields = self.dict_class()
        self.context = context or {}
        self._normalize_nested_options()
        self._types_seen = set()
        self._update_fields(many=many)

    def __repr__(self):
        return '<{ClassName}(many={self.many})>'.format(
            ClassName=self.__class__.__name__, self=self,
        )

    @property
    def dict_class(self):
        return OrderedDict if self.ordered else dict

    @property
    def set_class(self):
        return OrderedSet if self.ordered else set

    ##### Override-able methods #####

    def handle_error(self, error, data):
        """Custom error handler function for the schema.

        :param ValidationError error: The `ValidationError` raised during (de)serialization.
        :param data: The original input data.

        .. versionadded:: 2.0.0
        """
        pass

    def get_attribute(self, obj, attr, default):
        """Defines how to pull values from an object to serialize.

        .. versionadded:: 2.0.0

        .. versionchanged:: 3.0.0a1
            Changed position of ``obj`` and ``attr``.
        """
        return utils.get_value(obj, attr, default)

    ##### Serialization/Deserialization API #####

    def dump(self, obj, many=None, update_fields=True):
        """Serialize an object to native Python data types according to this
        Schema's fields.

        :param obj: The object to serialize.
        :param bool many: Whether to serialize `obj` as a collection. If `None`, the value
            for `self.many` is used.
        :param bool update_fields: Whether to update the schema's field classes. Typically
            set to `True`, but may be `False` when serializing a homogenous collection.
            This parameter is used by `fields.Nested` to avoid multiple updates.
        :return: A dict of serialized data
        :rtype: dict

        .. versionadded:: 1.0.0
        .. versionchanged:: 3.0.0b7
            This method returns the serialized data rather than a ``(data, errors)`` duple.
            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised
            if ``obj`` is invalid.
        """
        # Callable marshalling object
        marshal = marshalling.Marshaller(prefix=self.prefix)
        errors = {}
        many = self.many if many is None else bool(many)
        if many and utils.is_iterable_but_not_string(obj):
            obj = list(obj)

        if self._has_processors(PRE_DUMP):
            try:
                processed_obj = self._invoke_dump_processors(
                    PRE_DUMP,
                    obj,
                    many,
                    original_data=obj,
                )
            except ValidationError as error:
                errors = error.normalized_messages()
                result = None
        else:
            processed_obj = obj

        if not errors:
            if update_fields:
                obj_type = type(processed_obj)
                if obj_type not in self._types_seen:
                    self._update_fields(processed_obj, many=many)
                    if not isinstance(processed_obj, Mapping):
                        self._types_seen.add(obj_type)

            try:
                result = marshal(
                    processed_obj,
                    self.fields,
                    many=many,
                    accessor=self.get_attribute,
                    dict_class=self.dict_class,
                    index_errors=self.opts.index_errors,
                )
            except ValidationError as error:
                errors = marshal.errors
                result = error.data

        if not errors and self._has_processors(POST_DUMP):
            try:
                result = self._invoke_dump_processors(
                    POST_DUMP,
                    result,
                    many,
                    original_data=obj,
                )
            except ValidationError as error:
                errors = error.normalized_messages()
        if errors:
            exc = ValidationError(
                errors,
                field_names=marshal.error_field_names,
                data=obj,
                valid_data=result,
                **marshal.error_kwargs
            )
            # User-defined error handler
            self.handle_error(exc, obj)
            raise exc

        return result

    def dumps(self, obj, many=None, update_fields=True, *args, **kwargs):
        """Same as :meth:`dump`, except return a JSON-encoded string.

        :param obj: The object to serialize.
        :param bool many: Whether to serialize `obj` as a collection. If `None`, the value
            for `self.many` is used.
        :param bool update_fields: Whether to update the schema's field classes. Typically
            set to `True`, but may be `False` when serializing a homogenous collection.
            This parameter is used by `fields.Nested` to avoid multiple updates.
        :return: A ``json`` string
        :rtype: str

        .. versionadded:: 1.0.0
        .. versionchanged:: 3.0.0b7
            This method returns the serialized data rather than a ``(data, errors)`` duple.
            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised
            if ``obj`` is invalid.
        """
        serialized = self.dump(obj, many=many, update_fields=update_fields)
        return self.opts.render_module.dumps(serialized, *args, **kwargs)

    def load(self, data, many=None, partial=None, unknown=None):
        """Deserialize a data structure to an object defined by this Schema's
        fields and :meth:`make_object`.

        :param dict data: The data to deserialize.
        :param bool many: Whether to deserialize `data` as a collection. If `None`, the
            value for `self.many` is used.
        :param bool|tuple partial: Whether to ignore missing fields. If `None`,
            the value for `self.partial` is used. If its value is an iterable,
            only missing fields listed in that iterable will be ignored.
        :param unknown: Whether to exclude, include, or raise an error for unknown
            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.
            If `None`, the value for `self.unknown` is used.
        :return: A dict of deserialized data
        :rtype: dict

        .. versionadded:: 1.0.0
        .. versionchanged:: 3.0.0b7
            This method returns the deserialized data rather than a ``(data, errors)`` duple.
            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised
            if invalid data are passed.
        """
        return self._do_load(
            data, many, partial=partial, unknown=unknown,
            postprocess=True,
        )

    def loads(
        self, json_data, many=None, partial=None, unknown=None,
        **kwargs
    ):
        """Same as :meth:`load`, except it takes a JSON string as input.

        :param str json_data: A JSON string of the data to deserialize.
        :param bool many: Whether to deserialize `obj` as a collection. If `None`, the
            value for `self.many` is used.
        :param bool|tuple partial: Whether to ignore missing fields. If `None`,
            the value for `self.partial` is used. If its value is an iterable,
            only missing fields listed in that iterable will be ignored.
        :param unknown: Whether to exclude, include, or raise an error for unknown
            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.
            If `None`, the value for `self.unknown` is used.
        :return: A dict of deserialized data
        :rtype: dict

        .. versionadded:: 1.0.0
        .. versionchanged:: 3.0.0b7
            This method returns the deserialized data rather than a ``(data, errors)`` duple.
            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised
            if invalid data are passed.
        """
        data = self.opts.render_module.loads(json_data, **kwargs)
        return self.load(data, many=many, partial=partial, unknown=unknown)

    def validate(self, data, many=None, partial=None):
        """Validate `data` against the schema, returning a dictionary of
        validation errors.

        :param dict data: The data to validate.
        :param bool many: Whether to validate `data` as a collection. If `None`, the
            value for `self.many` is used.
        :param bool|tuple partial: Whether to ignore missing fields. If `None`,
            the value for `self.partial` is used. If its value is an iterable,
            only missing fields listed in that iterable will be ignored.
        :return: A dictionary of validation errors.
        :rtype: dict

        .. versionadded:: 1.1.0
        """
        try:
            self._do_load(data, many, partial=partial, postprocess=False)
        except ValidationError as exc:
            return exc.messages
        return {}

    ##### Private Helpers #####

    def _do_load(
        self, data, many=None, partial=None, unknown=None,
        postprocess=True,
    ):
        """Deserialize `data`, returning the deserialized result.

        :param data: The data to deserialize.
        :param bool many: Whether to deserialize `data` as a collection. If `None`, the
            value for `self.many` is used.
        :param bool|tuple partial: Whether to validate required fields. If its value is an iterable,
            only fields listed in that iterable will be ignored will be allowed missing.
            If `True`, all fields will be allowed missing.
            If `None`, the value for `self.partial` is used.
        :param unknown: Whether to exclude, include, or raise an error for unknown
            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.
            If `None`, the value for `self.unknown` is used.
        :param bool postprocess: Whether to run post_load methods..
        :return: A dict of deserialized data
        :rtype: dict
        """
        # Callable unmarshalling object
        unmarshal = marshalling.Unmarshaller()
        errors = {}
        many = self.many if many is None else bool(many)
        unknown = unknown or self.unknown
        if partial is None:
            partial = self.partial
        if self._has_processors(PRE_LOAD):
            try:
                processed_data = self._invoke_load_processors(
                    PRE_LOAD,
                    data,
                    many,
                    original_data=data,
                )
            except ValidationError as err:
                errors = err.normalized_messages()
                result = None
        else:
            processed_data = data
        if not errors:
            try:
                result = unmarshal(
                    processed_data,
                    self.fields,
                    many=many,
                    partial=partial,
                    unknown=unknown,
                    dict_class=self.dict_class,
                    index_errors=self.opts.index_errors,
                )
            except ValidationError as error:
                result = error.data
            self._invoke_field_validators(unmarshal, data=result, many=many)
            errors = unmarshal.errors
            # Run schema-level validation.
            if self._has_processors(VALIDATES_SCHEMA):
                field_errors = bool(errors)
                try:
                    self._invoke_schema_validators(
                        unmarshal,
                        pass_many=True,
                        data=result,
                        original_data=data,
                        many=many,
                        field_errors=field_errors,
                    )
                except ValidationError as err:
                    errors.update(err.messages)
                try:
                    self._invoke_schema_validators(
                        unmarshal,
                        pass_many=False,
                        data=result,
                        original_data=data,
                        many=many,
                        field_errors=field_errors,
                    )
                except ValidationError as err:
                    errors.update(err.messages)
        # Run post processors
        if not errors and postprocess and self._has_processors(POST_LOAD):
            try:
                result = self._invoke_load_processors(
                    POST_LOAD,
                    result,
                    many,
                    original_data=data,
                )
            except ValidationError as err:
                errors = err.normalized_messages()
        if errors:
            exc = ValidationError(
                errors,
                field_names=unmarshal.error_field_names,
                data=data,
                valid_data=result,
                **unmarshal.error_kwargs
            )
            self.handle_error(exc, data)
            raise exc

        return result

    def _normalize_nested_options(self):
        """Apply then flatten nested schema options"""
        if self.only is not None:
            # Apply the only option to nested fields.
            self.__apply_nested_option('only', self.only, 'intersection')
            # Remove the child field names from the only option.
            self.only = self.set_class(
                [field.split('.', 1)[0] for field in self.only],
            )
        excludes = set(self.opts.exclude) | set(self.exclude)
        if excludes:
            # Apply the exclude option to nested fields.
            self.__apply_nested_option('exclude', excludes, 'union')
        if self.exclude:
            # Remove the parent field names from the exclude option.
            self.exclude = self.set_class(
                [field for field in self.exclude if '.' not in field],
            )
        if self.opts.exclude:
            # Remove the parent field names from the meta exclude option.
            self.opts.exclude = self.set_class(
                [field for field in self.opts.exclude if '.' not in field],
            )

    def __apply_nested_option(self, option_name, field_names, set_operation):
        """Apply nested options to nested fields"""
        # Split nested field names on the first dot.
        nested_fields = [name.split('.', 1) for name in field_names if '.' in name]
        # Partition the nested field names by parent field.
        nested_options = defaultdict(list)
        for parent, nested_names in nested_fields:
            nested_options[parent].append(nested_names)
        # Apply the nested field options.
        for key, options in iter(nested_options.items()):
            new_options = self.set_class(options)
            original_options = getattr(self.declared_fields[key], option_name, ())
            if original_options:
                if set_operation == 'union':
                    new_options |= self.set_class(original_options)
                if set_operation == 'intersection':
                        new_options &= self.set_class(original_options)
            setattr(self.declared_fields[key], option_name, new_options)

    def _update_fields(self, obj=None, many=False):
        """Update fields based on the passed in object."""
        if self.opts.fields:
            # Return fields specified in fields option
            field_names = self.set_class(self.opts.fields)
        elif self.opts.additional:
            # Return declared fields + additional fields
            field_names = (self.set_class(self.declared_fields.keys()) |
                            self.set_class(self.opts.additional))
        else:
            field_names = self.set_class(self.declared_fields.keys())

        declared_fields = field_names
        invalid_fields = self.set_class()
        if self.only is not None:
            # Return only fields specified in only option
            only = self.set_class(self.only)
            field_names = only
            invalid_only = only - declared_fields
            invalid_fields |= invalid_only

        # If "exclude" option or param is specified, remove those fields
        excludes = set(self.opts.exclude) | set(self.exclude)
        if excludes:
            field_names = field_names - excludes
            invalid_excludes = excludes - declared_fields
            invalid_fields |= invalid_excludes

        if invalid_fields:
            message = 'Invalid fields for {0}: {1}.'.format(self, invalid_fields)
            raise ValueError(message)

        ret = self.__filter_fields(field_names, obj, many=many)
        # Set parents
        self.__set_field_attrs(ret)
        self.fields = ret
        return self.fields

    def on_bind_field(self, field_name, field_obj):
        """Hook to modify a field when it is bound to the `Schema`. No-op by default."""
        return None

    def __set_field_attrs(self, fields_dict):
        """Bind fields to the schema, setting any necessary attributes
        on the fields (e.g. parent and name).

        Also set field load_only and dump_only values if field_name was
        specified in ``class Meta``.
        """
        for field_name, field_obj in iteritems(fields_dict):
            try:
                if field_name in self.load_only:
                    field_obj.load_only = True
                if field_name in self.dump_only:
                    field_obj.dump_only = True
                field_obj._add_to_schema(field_name, self)
                self.on_bind_field(field_name, field_obj)
            except TypeError:
                # field declared as a class, not an instance
                if (isinstance(field_obj, type) and
                        issubclass(field_obj, base.FieldABC)):
                    msg = ('Field for "{0}" must be declared as a '
                           'Field instance, not a class. '
                           'Did you mean "fields.{1}()"?'
                           .format(field_name, field_obj.__name__))
                    raise TypeError(msg)
        return fields_dict

    def __filter_fields(self, field_names, obj, many=False):
        """Return only those field_name:field_obj pairs specified by
        ``field_names``.

        :param set field_names: Field names to include in the final
            return dictionary.
        :returns: An dict of field_name:field_obj pairs.
        """
        if obj and many:
            try:  # Homogeneous collection
                # Prefer getitem over iter to prevent breaking serialization
                # of objects for which iter will modify position in the collection
                # e.g. Pymongo cursors
                if hasattr(obj, '__getitem__') and callable(getattr(obj, '__getitem__')):
                    try:
                        obj_prototype = obj[0]
                    except KeyError:
                        obj_prototype = next(iter(obj))
                else:
                    obj_prototype = next(iter(obj))
            except (StopIteration, IndexError):  # Nothing to serialize
                return {k: v for k, v in self.declared_fields.items() if k in field_names}
            obj = obj_prototype
        ret = self.dict_class()
        for key in field_names:
            if key in self.declared_fields:
                ret[key] = self.declared_fields[key]
            else:  # Implicit field creation (class Meta 'fields' or 'additional')
                if obj:
                    attribute_type = None
                    try:
                        if isinstance(obj, Mapping):
                            attribute_type = type(obj[key])
                        else:
                            attribute_type = type(getattr(obj, key))
                    except (AttributeError, KeyError) as err:
                        err_type = type(err)
                        raise err_type(
                            '"{0}" is not a valid field for {1}.'.format(key, obj),
                        )
                    field_obj = self.TYPE_MAPPING.get(attribute_type, fields.Field)()
                else:  # Object is None
                    field_obj = fields.Field()
                # map key -> field (default to Raw)
                ret[key] = field_obj
        return ret

    def _has_processors(self, tag):
        return self._hooks[(tag, True)] or self._hooks[(tag, False)]

    def _invoke_dump_processors(self, tag, data, many, original_data=None):
        # The pass_many post-dump processors may do things like add an envelope, so
        # invoke those after invoking the non-pass_many processors which will expect
        # to get a list of items.
        data = self._invoke_processors(
            tag, pass_many=False,
            data=data, many=many, original_data=original_data,
        )
        data = self._invoke_processors(
            tag, pass_many=True,
            data=data, many=many, original_data=original_data,
        )
        return data

    def _invoke_load_processors(self, tag, data, many, original_data=None):
        # This has to invert the order of the dump processors, so run the pass_many
        # processors first.
        data = self._invoke_processors(
            tag, pass_many=True,
            data=data, many=many, original_data=original_data,
        )
        data = self._invoke_processors(
            tag, pass_many=False,
            data=data, many=many, original_data=original_data,
        )
        return data

    def _invoke_field_validators(self, unmarshal, data, many):
        for attr_name in self._hooks[VALIDATES]:
            validator = getattr(self, attr_name)
            validator_kwargs = validator.__marshmallow_hook__[VALIDATES]
            field_name = validator_kwargs['field_name']

            try:
                field_obj = self.fields[field_name]
            except KeyError:
                if field_name in self.declared_fields:
                    continue
                raise ValueError('"{0}" field does not exist.'.format(field_name))

            if many:
                for idx, item in enumerate(data):
                    try:
                        value = item[field_obj.attribute or field_name]
                    except KeyError:
                        pass
                    else:
                        validated_value = unmarshal.call_and_store(
                            getter_func=validator,
                            data=value,
                            field_name=field_obj.data_key or field_name,
                            index=(idx if self.opts.index_errors else None),
                        )
                        if validated_value is missing:
                            data[idx].pop(field_name, None)
            else:
                try:
                    value = data[field_obj.attribute or field_name]
                except KeyError:
                    pass
                else:
                    validated_value = unmarshal.call_and_store(
                        getter_func=validator,
                        data=value,
                        field_name=field_obj.data_key or field_name,
                    )
                    if validated_value is missing:
                        data.pop(field_name, None)

    def _invoke_schema_validators(
        self,
        unmarshal,
        pass_many,
        data,
        original_data,
        many,
        field_errors=False,
    ):
        errors = {}
        for attr_name in self._hooks[(VALIDATES_SCHEMA, pass_many)]:
            validator = getattr(self, attr_name)
            validator_kwargs = validator.__marshmallow_hook__[(VALIDATES_SCHEMA, pass_many)]
            pass_original = validator_kwargs.get('pass_original', False)

            skip_on_field_errors = validator_kwargs['skip_on_field_errors']
            if skip_on_field_errors and field_errors:
                continue

            if pass_many:
                validator = functools.partial(validator, many=many)
            if many and not pass_many:
                for idx, (item, orig) in enumerate(zip(data, original_data)):
                    try:
                        unmarshal.run_validator(
                            validator,
                            item,
                            orig,
                            self.fields,
                            many=many,
                            index=idx,
                            pass_original=pass_original,
                        )
                    except ValidationError as err:
                        errors.update(err.messages)
            else:
                try:
                    unmarshal.run_validator(
                        validator,
                        data,
                        original_data,
                        self.fields,
                        many=many,
                        pass_original=pass_original,
                    )
                except ValidationError as err:
                    errors.update(err.messages)
        if errors:
            raise ValidationError(errors)
        return None

    def _invoke_processors(
        self,
        tag,
        pass_many,
        data,
        many,
        original_data=None,
    ):
        key = (tag, pass_many)
        for attr_name in self._hooks[key]:
            # This will be a bound method.
            processor = getattr(self, attr_name)

            processor_kwargs = processor.__marshmallow_hook__[key]
            pass_original = processor_kwargs.get('pass_original', False)

            if pass_many:
                if pass_original:
                    data = processor(data, many, original_data)
                else:
                    data = processor(data, many)
            elif many:
                if pass_original:
                    data = [
                        processor(item, original)
                        for item, original in zip(data, original_data)
                    ]
                else:
                    data = [processor(item) for item in data]
            else:
                if pass_original:
                    data = processor(data, original_data)
                else:
                    data = processor(data)
        return data

class Schema(with_metaclass(SchemaMeta, BaseSchema)):
    __doc__ = BaseSchema.__doc__


"""
https://leetcode.com/problems/palindrome-partitioning/description/
"""
from typing import List

class Solution:

    def dfs(self, s, i, temp, result):
        if i >= len(s):
            result.append(temp[:])
            return
        for j in range(i, len(s)):
            if s[i:j+1] == s[i:j+1][::-1]:
                temp.append(s[i:j+1])
                self.dfs(s, j+1, temp, result)
                temp.pop()
    def partition(self, s: str) -> List[List[str]]:
        result = []
        self.dfs(s, 0, [], result)
        return result

solve = Solution()
print(solve.partition(s='aab'))


import serial
import parameter as param
import numpy as np

class Serial_COM:
    def __init__(self, bit_rate, dec):
        self.dec = dec
        self.bit_rate = bit_rate
        self.val = []
        # self.ser = serial.Serial(param.com_port, bit_rate, timeout=1)
        # self.ser = serial.Serial(param.com_port, bit_rate, timeout=1)

    def get_mean_std(self, cycle, label):
        ser = serial.Serial(param.com_port, self.bit_rate, timeout=1)
        mean = 0
        std_dev = 0

        for i in range(cycle):
            if self.ser.readable():
                vals.append(int(self.ser.readline().decode(self.dec)[-3:]) / 10)
                if(param.VIS): print(f'{i}: {vals[-1]}')
        self.ser.close()

        mean = np.mean(vals)
        std_dev = np.std(vals)

        del vals
        ser.close()

        return mean, std_dev


# -*- coding: utf-8 -*-

from abc import ABCMeta, abstractmethod
import pandas as pd

from event import FillEvent
import datetime
import math

class ExecutionHandler(object, metaclass=ABCMeta):
    """
    ExecutionHandler抽象类处理由Portfolio生成的order对象
    与实际市场中发生的Fill对象之间的交互
    这个类可以用于实际的成交，或者模拟的成交
    """

    __metaclass__ = ABCMeta

    @abstractmethod
    def execute_order(self, event):
        """
        获取一个Order事件并执行，产生Fill事件放到事件队列中
        """
        raise NotImplementedError("Should implement execute_order()")

class SimulatedExecution(ExecutionHandler):
    """
    这是一个模拟的执行处理，简单的将所有的Order对象转化为等价的Fill对象
    不考虑时延，滑价以及成交比率的影响
    """

    def __init__(self, events, ContextInfo, bars, portfolio):
        self.events = events
        self.bars = bars
        self.portfolio = portfolio
        self.commission_rate = ContextInfo.commission_rate
        self.min_commission = ContextInfo.min_commission
        #self.execution_records = pd.DataFrame(
        #    columns=['date_time','symbol','order_price','order_amount','commission'])
        self.execution_records = []

    def update_temp_pos(self):
        """
        做一个临时的持仓表，以判断现金和持仓是否充足可用（实盘由交易所判断）
        在Portfolio处理完FILL事件后，重新获取temp_pos
        """
        #print(self.portfolio.positions)
        self.temp_pos = {}
        self.temp_pos['cash'] = self.portfolio.positions['cash']
        for sym in self.portfolio.symbol_list:
            self.temp_pos[sym] = self.portfolio.positions[sym]
        #print(self.temp_pos)
        #print(len(self.temp_pos))

    def execute_order(self, event):
        """
        接收Order事件，生成Fill事件
        """
        symbol = event.symbol
        order_amount = event.order_amount
        order_price = event.order_price

        cur_date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        bar_date = self.bars.get_latest_bar(symbol)['date_time']
        last_close = self.bars.get_latest_bar(symbol)['close']

        # 判断是否停牌
        if self.bars.is_suspended_stock(symbol):
            print("[%s][股票当日停牌]Time：%s，Symbol：%s，Stock Suspended！"%(cur_date,bar_date,symbol))
            return

        # 判断是否涨/跌停（注意：未考虑创业/科创板的20%涨跌停及ST股的5%涨跌停）
        # 如果涨停买入或跌停卖出，则委托无效，不进行撮合
        if self.bars.get_latest_bar(symbol)['pct_chg'] >= 9.98 and order_amount > 0:
            print("[%s][股票当日涨停]Time：%s，Symbol：%s，Trade Limited！"%(cur_date,bar_date,symbol))
            return
        if self.bars.get_latest_bar(symbol)['pct_chg'] <= -9.98 and order_amount < 0:
            print("[%s][股票当日跌停]Time：%s，Symbol：%s，Trade Limited！"%(cur_date,bar_date,symbol))
            return

        # 市价单：委托价为收盘价
        if order_price == 'MARKET':
            order_price = last_close

        # 股价判断：买入/卖出价若低于/高于收盘价，则按收盘价执行
        if order_amount < 0 and order_price > last_close:
            order_price = last_close
        if order_amount > 0 and order_price < last_close:
            order_price = last_close

        # 处理order_amount，向下取整百（需要用floor处理负数）
        order_amount = math.floor(order_amount/100)*100
        if order_amount == 0: # 委托数量为0，则中止运行
            print("[%s][委托数量错误]Time：%s，Symbol：%s，Price：%s，Qty：%s，Quantity Error！"%(cur_date,bar_date,symbol,order_price,order_amount))
            return

        # 计算手续费=MAX（交易额*佣金率，最低手续费）
        commission = max(round(order_price*abs(order_amount)*self.commission_rate,2),self.min_commission)

        # 判断现金及持仓是否可用
        if order_amount > 0: # 若买入，判断可用资金（暂不考虑手续费，现金可能出现负数）
            #if order_amount*order_price+commission > self.temp_pos['cash']:
            if order_amount*order_price > self.temp_pos['cash']:
                # 可用资金不足，不执行买入
                print("[%s][可用资金不足]Time：%s，Symbol：%s，Price：%s，Qty：%s，Cash Error！"%(cur_date,bar_date,symbol,order_price,order_amount))
                return
        elif order_amount < 0: # 若卖出，判断可用数量
            if order_amount < -self.temp_pos[symbol]:
                # 可用数量不足，拒绝执行卖出
                print("[%s][可用持仓不足]Time：%s，Symbol：%s，Price：%s，Qty：%s，Position Error！"%(cur_date,bar_date,symbol,order_price,order_amount))
                return

        # 交易撮合，生成FILL事件
        fill_event = FillEvent(symbol,order_amount,order_price,commission)
        self.events.put(fill_event)
        print("[%s][委托交易成功]Time：%s，Symbol：%s，Price：%s，Qty：%s"%(cur_date,bar_date,symbol,order_price,order_amount))

        # 实时更新至temp_pos，以便验证每个订单的持仓及金额是否可用
        # 此处遇到问题：如果直接令temp_pos=portfolio.positions，
        # 当temp_pos变动时，portfolio也会发生变动，
        # 而新建一个temp_pos字典可以解决这个问题，具体原因不明
        # print(self.portfolio.positions)
        self.temp_pos['cash'] -= order_price*order_amount+commission
        self.temp_pos[symbol] += order_amount
        # print(self.portfolio.positions)

        # 写入交易记录
        record = {}
        record['date_time'] = bar_date
        record['symbol'] = symbol
        record['order_price'] = order_price
        record['order_amount'] = order_amount
        record['commission'] = commission
        #self.execution_records = self.execution_records.append(record, ignore_index=True)
        self.execution_records.append(record)

    def download_records(self):
        """
        导出交易记录
        """
        execution_records = pd.DataFrame(self.execution_records)
        execution_records.to_csv('logs/execution_records.csv')


from runner import Runner
from data.data_loader import DataLoader
from ekphrasis.classes.preprocessor import TextPreProcessor
from ekphrasis.classes.tokenizer import SocialTokenizer
from ekphrasis.dicts.emoticons import emoticons
from etc.logger import Logger
from feature_extraction.feature_extractor import FeatureExtractor
import random as rn
import numpy as np
import tensorflow as tf
import os

# add seeds for reproducibility
SEED = 1337
os.environ['PYTHONHASHSEED'] = '0'
np.random.seed(SEED)
rn.seed(SEED)
tf.set_random_seed(SEED)

TERNARY = True

# preprocessor for training/test
preprocessor = TextPreProcessor(
    normalize=['url', 'email', 'percent', 'money', 'phone', 'user', 'time',
               'date', 'number'],
    annotate={'hashtag', 'allcaps', 'elongated', "repeated", 'emphasis',
              'censored'},
    fix_html=True,
    segmenter="twitter",
    corrector="twitter",
    unpack_hashtags=True,
    unpack_contractions=True,
    spell_correct_elong=False,
    tokenizer=SocialTokenizer(lowercase=True).tokenize,
    dicts=[emoticons])

# preprocessor for features extracting
feature_preprocessor = TextPreProcessor(
    normalize=['url', 'email', 'percent', 'money', 'phone', 'user', 'time',
               'date', 'number'],
    fix_html=True,
    segmenter='twitter',
    corrector='twitter',
    unpack_contractions=True,
    spell_correct_elong=False,
    tokenizer=SocialTokenizer(lowercase=True).tokenize,
    dicts=[emoticons])

logger = Logger()
runner = Runner(logger=logger, ternary=TERNARY,
                model_type='baseline', use_embeddings=True)

logger.write('preprocessing: %s' % (True if preprocessor else False))

data_loader = DataLoader(preprocessor=preprocessor)
train, test = data_loader.get_train_test(ternary=TERNARY)
extra_train = data_loader.get_train(ternary=TERNARY, \
    paths=['data/ydata-ynacc-v1_0_expert_annotations_filt.tsv'])

feature_data_loader = DataLoader(preprocessor=feature_preprocessor)
feature_extractor = FeatureExtractor(data_loader=feature_data_loader,
                                     logger=logger)

train_feats, test_feats = feature_extractor.get_train_test_features(
    ternary=TERNARY,
    manual=True,
    auto=True,
    scaled=False)

runner.run(train, test, extra_train=extra_train)


#!/usr/bin/env python3
import argparse
import sys

import vyper
from vyper.codegen.ir_node import IRnode
from vyper.ir import compile_ir, optimizer
from vyper.ir.s_expressions import parse_s_exp

def _parse_cli_args():
    return _parse_args(sys.argv[1:])

def _parse_args(argv):
    parser = argparse.ArgumentParser(description="Vyper IR IR compiler")
    parser.add_argument("input_file", help="Vyper sourcecode to compile")
    parser.add_argument(
        "--version", action="version", version=f"{vyper.__version__}+commit{vyper.__commit__}"
    )
    parser.add_argument(
        "-f",
        help="Format to print csv list of ir,opt_ir,asm,bytecode",
        default="bytecode",
        dest="format",
    )
    parser.add_argument(
        "--show-gas-estimates", help="Show gas estimates in ir output mode.", action="store_true"
    )

    args = parser.parse_args(argv)
    output_formats = set(dict.fromkeys(args.format.split(",")))
    compiler_data = compile_to_ir(args.input_file, output_formats, args.show_gas_estimates)

    for key in ("ir", "opt_ir", "asm", "bytecode"):
        if key in compiler_data:
            print(compiler_data[key])

def compile_to_ir(input_file, output_formats, show_gas_estimates=False):
    with open(input_file) as fh:
        s_expressions = parse_s_exp(fh.read())

    if show_gas_estimates:
        IRnode.repr_show_gas = True

    compiler_data = {}
    ir = IRnode.from_list(s_expressions[0])
    ir = optimizer.optimize(ir)
    if "ir" in output_formats:
        compiler_data["ir"] = ir

    asm = compile_ir.compile_to_assembly(ir)
    if "asm" in output_formats:
        compiler_data["asm"] = asm

    if "bytecode" in output_formats:
        bytecode, _ = compile_ir.assembly_to_evm(asm)
        compiler_data["bytecode"] = "0x" + bytecode.hex()

    return compiler_data

if __name__ == "__main__":
    _parse_cli_args()


import torch
import torch.nn as nn
import torch.nn.init as init
import torch.nn.functional as F
import numpy as np
import random
import sys
import scipy.spatial.distance
import math
import random
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils
import os
import open3d as o3d
import time
from torch_geometric.nn import knn_interpolate, fps
import json
from collections import Counter, OrderedDict
from torch.optim import AdamW
from loss import *
from utils import *
from model import *

batch_size = 2
batch_size_test = 2

def train_all(model,disc,train_loader,valid_loader, n_cls,count=None, shape = None, epochs=500, smoothing= False, save=True):
    best = 0
    patience = 100
    p = 0
    if smoothing:
        sig = 0.0
    else:
        sig = 0.0
    log_var_a = torch.zeros((1,), requires_grad=True)
    log_var_b = torch.zeros((1,), requires_grad=True)

    params = ([p for p in model.parameters()] + [log_var_a] + [log_var_b])
    optimizer = torch.optim.Adam(params, lr=lr)
    optimizerD = torch.optim.Adam(disc.parameters(), lr=lr2)
    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, step*len(train_loader), eta_min=1e-4)
    schedulerD = torch.optim.lr_scheduler.CosineAnnealingLR(optimizerD, step*len(train_loader), eta_min=1e-4)

    for epoch in range(epochs):
        e = epoch
        model.train()
        running_loss = 0
        for i, data in enumerate(train_loader):
            inputs, labels = data['points'].to(device), data['labels'].to(device)
            #UPDATE DISCRIMINATOR
            optimizerD.zero_grad()

            labels = labels.view(batch_size, -1)

            labels2 = one_hot(labels,n_cls).to(device)
            noise = torch.tensor(np.random.normal(0, sig, labels2.shape)).to(device)
            labels2 = labels2+noise

            inputs2 = torch.cat((inputs, labels2.type(torch.float64).reshape(batch_size,2048,n_cls)),axis=2)
            inputs2 = inputs2.transpose(1,2)
            real_image = inputs2
            output,em1 = disc(inputs2.float())
            ls_real = disc_loss(output,num=1, smoothing = smoothing)
            ls_real.backward(retain_graph=True)

            #Fake image
            with torch.no_grad():
                output = model(inputs.transpose(1,2))
            output_pred = output
            _,pred = torch.max(output.data,1)
            pred = pred.view(batch_size,-1)
            pred = one_hot(pred,n_cls)
            noise = torch.tensor(np.random.normal(0, sig, pred.shape))#.to(device)
            pred += noise
            pred = pred.to(device)
            inputs2 = torch.cat((inputs, pred.type(torch.float64).reshape(batch_size,2048,n_cls)),axis=2)
            inputs2 = inputs2.transpose(1,2)
            output,em2 = disc(inputs2.float())
            ls_fake = disc_loss(output, num=0, smoothing = smoothing)
            ls_fake.backward()

            ls_D = (ls_real + ls_fake)/2
            optimizerD.step()
            schedulerD.step()

            #UPDATE GENERATOR
            #Real image
            optimizer.zero_grad()
            #real
            inputs2 = torch.cat((inputs, labels2.type(torch.float64).reshape(batch_size,2048,n_cls)),axis=2)
            inputs2 = inputs2.transpose(1,2)
            real_image = inputs2
            #fake
            output = model(inputs.transpose(1,2))
            output_pred = output
            output2 = output.clone()
            _,pred = torch.max(output.data,1)
            pred = pred.view(batch_size,-1)
            pred = one_hot(pred,n_cls)
            noise = torch.tensor(np.random.normal(0, sig, pred.shape))#.to(device)
            pred += noise
            pred = pred.to(device)
            inputs2 = torch.cat((inputs, pred.type(torch.float64).reshape(batch_size,2048,n_cls)),axis=2)
            inputs2 = inputs2.transpose(1,2)

            _,em1 = disc(real_image.float())
            _,em2 = disc(inputs2.float())

            output_pred = [output_pred,em2]
            labels = [labels,em1]
            total_ls, emb_ls = multitask_loss(output_pred,labels,n_cls,[log_var_a,log_var_b])
            total_ls.backward()

            #if i%30 == 0:
            #    print("Embedding Loss: {}".format(emb_ls))
            optimizer.step()
            scheduler.step()

        std_1 = torch.exp(-log_var_a)**0.5
        std_2 = torch.exp(-log_var_b)**0.5

        model.eval()
        total = 0
        correct =0
        total2 = 0
        correct2 =0
        loss =[]
        loss2 = []
        valid_iou = []
        with torch.no_grad():
            for i, data in enumerate(valid_loader):
                inputs, labels = data['points'].to(device), data['labels'].to(device)

                #Discriminator
                #Real
                labels = labels.view(batch_size, -1)

                labels2 = one_hot(labels,n_cls).to(device)
                noise = torch.tensor(np.random.normal(0, sig, labels2.shape)).to(device)
                labels2 = labels2+noise

                inputs2 = torch.cat((inputs, labels2.type(torch.float64).reshape(batch_size,2048,n_cls)),axis=2)
                inputs2 = inputs2.transpose(1,2)
                real_image = inputs2
                output,em1 = disc(inputs2.float())
                _,pred2 = torch.max(output.data,1)
                ls_real = disc_loss(output,num=1)
                total2 += batch_size_test
                correct2 += (pred2 == 1).sum().item()

                #Fake
                with torch.no_grad():
                    outputs = model(inputs.transpose(1,2))
                output_pred = outputs
                _,pred = torch.max(outputs.data,1)
                pred = pred.view(batch_size,-1)
                pred = one_hot(pred,n_cls)
                noise = torch.tensor(np.random.normal(0, sig, pred.shape))#.to(device)
                pred += noise
                pred = pred.to(device)
                inputs2 = torch.cat((inputs, pred.type(torch.float64).reshape(batch_size,2048,n_cls)),axis=2)
                inputs2 = inputs2.transpose(1,2)
                output,em2 = disc(inputs2.float())
                _,pred2 = torch.max(output.data,1)
                ls_fake = disc_loss(output, num=0)

                ls_total = (ls_real + ls_fake)/2
                loss2.append(ls_total.item())
                total2 += batch_size_test
                correct2 += (pred2 == 0).sum().item()

                #Generator
                outputs = model(inputs.transpose(1,2))
                ls = gagcn_loss(outputs,labels,n_cls, smoothing = True)

                loss.append(ls.item())
                _,pred = torch.max(outputs.data,1) #why? because the first return is for the values(probabilities)
                total += labels.size(0) * labels.size(1)
                correct += (pred == labels).sum().item()
                pred = pred.cpu().detach().numpy()
                ious = calculate_shape_IoU(pred,labels.cpu().detach().numpy(),n_cls)
                valid_iou.append(np.mean(ious))

            #Generaotr
            acc = 100. * correct / total
            loss_final = np.mean(loss)
            iou_final = np.mean(valid_iou)
            print("Epoch: {}".format(e))
            print("Generator: The validation accuracy: {:.6f} and the validation loss: {:.6f} and the validation iou: {:.6f}".format(acc,loss_final,iou_final))
            print("std1 {} std2 {}".format(std_1,std_2))
            #Discriminator
            acc = 100. * correct2 / total2
            loss_final = np.mean(loss2)
            print("Discriminator: The validation accuracy: {:.6f} and the validation loss: {:.6f}".format(acc,loss_final))

            if best < iou_final:
                best = iou_final
                p = 0
                if save:
                    torch.save(model.state_dict(), "/home/edshkim98/shkim/pretrained_group_final/partseg_"+str(shape)+'_'+str(count)+".pth")
                else:
                    pass
            else:
                p +=1
                print("Patience: {} / {}".format(p,patience))
                if p == patience:
                    print("Stopped due to convergence")
                    return best
            print("Valid best iou: ",best)
    return best

if __name__ == "__main__":
    dataset = {"02691156": "Airplane", "02773838": "Bag", "02954340": "Cap", "02958343": "Car", "03001627": "Chair",
           "03261776": "Earphone", "03467517": "Guitar", "03624134": "Knife", "03636649": "Lamp", "03642806": "Laptop",
           "03790512": "Motorbike", "03797390": "Mug", "03948459": "Pistol", "04099429": "Rocket", "04225987": "Skateboard",
           "04379243": "Table"}
    seg_num = [4, 2, 2, 4, 4, 3, 3, 2, 4, 2, 6, 2, 3, 3, 3, 3]
    print("Training on ShapeNet dataset")
    print("#############################")
    total_final = 0
    total_num = 0
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    for o in range(15,len(dataset.keys())):
        shape = list(dataset.keys())[-3]
        path = r'/home/edshkim98/shkim/shape_data/'+shape+'/'#+shape
        new_path =path+'/points_label'
        n_cls = seg_num[-3]

        valid_files = '/home/edshkim98/shkim/shape_data/train_test_split/shuffled_val_file_list.json'
        train_files = '/home/edshkim98/shkim/shape_data/train_test_split/shuffled_train_file_list.json'
        test_files = '/home/edshkim98/shkim/shape_data/train_test_split/shuffled_test_file_list.json'

        torch.manual_seed(42)
        torch.cuda.manual_seed(42)
        random.seed(42)
        np.random.seed(42)

        train_files = read_json(train_files)
        val_files = read_json(valid_files)
        test_files = read_json(test_files)

        train_xs = []
        val = []
        test = []
        for i in train_files:
            if shape in i:
                train_xs.append(i)
        for i in val_files:
            if shape in i:
                val.append(i)
        for i in test_files:
            if shape in i:
                test.append(i)
        total = train_xs+val+test
        np.random.shuffle(total)
        #Train
        train_xs = train_xs+val#np.array(total)[idx]
        #Val&Test
        val = test
        train_dataset = CustomDataset(path,train_xs, transform =train_transforms,valid = False)
        val_dataset = CustomDataset(path, val,transform=None, valid=True)

        train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last = True, num_workers =2)
        val_loader = DataLoader(val_dataset, batch_size=batch_size_test, shuffle=False, drop_last = True, num_workers =2)

        print('######### Dataset class created #########')
        print("Shape: {} Batch size: {}".format(shape,batch_size))
        print('Number of images: ', len(train_xs)+len(val))
        print('Sample image shape: ', train_dataset[0]['points'].shape)
        print('train size in no of batch: ',len(train_loader))
        print("test size in no of batch: ",len(val_loader))
        print('train size: ',len(train_loader)*batch_size)
        print("valid size: ",len(val_loader)*batch_size_test)

            #########################

            ###########################
        best = 0
        total = len(val_loader)*batch_size_test

        if len(train_loader)*batch_size > 1500:
            cnts = 1
        else:
            cnts=2

        for cnt in range(cnts):
            smoothing=False
            bias = True
            dilation_rate = [1,3,6,6,6,1]
            pyramid = [False,False,False,False,False]
            dilation = 'order'
            lr = 0.005
            lr2 = 0.01
            step = 5

            se = False
            gagcn = GAGCN(n_cls,k=16,pyramid=pyramid,dilation='order', dilation_rate = dilation_rate, norm='group', bias= bias)
            torch.cuda.set_device(0)
            gagcn.to(device)
            pytorch_total_params = sum(p.numel() for p in gagcn.parameters())
            print("Number of parameters (segmentation): ", pytorch_total_params)

            se = False
            disc = Discriminator(k=12, dilation='order', dilation_rate= [1,3,3,3], n_cls= n_cls, out_dims= [128,256,256,512],pyramid=[False,False,False,False],norm='group')
            disc.to(device)
            pytorch_total_params = sum(p.numel() for p in disc.parameters())
            print("Number of parameters (discriminator): ", pytorch_total_params)
            print("Disc smoothing: ",smoothing)

            best_curr = train_all(gagcn,disc,train_loader, val_loader, n_cls, count=cnt,shape=shape, smoothing = smoothing,save=False)
            if best_curr > best:
                best = best_curr
                print("###############################################")
                print("Best Setting-> Shape {} IoU {} count {}".format(shape, best, cnt))
                print("Best: {}%".format(best*100))
                print("###############################################")


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def findTarget(self, root: TreeNode, k: int) -> bool:
        x=[]
        stack=[root]
        while stack:
            node = stack.pop()
            if node:
                x.append(node.val)
                stack.append(node.left)
                stack.append(node.right)
        for i in range(len(x)):
            if k-x[i] in x and (k-x[i])!=x[i]:
                return True

Trees/653. Two Sum IV - Input is a BST.py


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Configuration
import os
import sys
file_path = os.path.dirname(os.path.abspath(__file__))
config_path = os.path.sep.join(file_path.split(os.path.sep)[:-1])
sys.path.append(config_path)

import itertools
import pickle as pk
from tqdm import tqdm
from collections import Counter

from util import *
scci_path = SCCIPath()

from connlp.preprocess import Normalizer, KoreanTokenizer
normalizer = Normalizer()
tokenizer = KoreanTokenizer(pre_trained=True, analyzer='Hannanum')

def data_import():
    flist = os.listdir(scci_path.fdir_article)
    docs = []
    for fname in flist:
        fpath = os.path.join(scci_path.fdir_article, fname)
        with open(fpath, 'rb') as f:
            docs.append(pk.load(f))
    return docs

def tokenize_docs(docs):
    normalized = [normalizer.normalize(a.content) for a in docs]

    tokenized = []
    with tqdm(total=len(normalized)) as pbar:
        for doc in normalized:
            tokenized.append(tokenizer.extract_noun(doc))
            pbar.update(1)
    return tokenized

def load_stoplist(fname):
    fpath = os.path.join(scci_path.fdir_stoplist, fname)
    with open(fpath, 'r', encoding='utf-8') as f:
        stoplist = list(set([w.strip() for w in f.read().strip().split('\n')]))

    with open(fpath, 'w', encoding='utf-8') as f:
        f.write('\n'.join(stoplist))

    return stoplist

def count_words(docs, fname_tokenized, do_tokenize, fname_stoplist):
    print('========================================')
    print('Tokenize articles')
    fpath_tokenized = os.path.join(scci_path.fdir_tokenized, fname_tokenized)
    if do_tokenize:
        tokenized = tokenize_docs(docs)
        makedir(fpath_tokenized)
        with open(fpath_tokenized, 'wb') as f:
            pk.dump(tokenized, f)
    else:
        with open(fpath_tokenized, 'rb') as f:
            tokenized = pk.dump(f)

    print('========================================')
    print('Stopword removal')
    stoplist = load_stoplist(fname_stoplist)
    stopword_removed = []
    with tqdm(total=len(tokenized)) as pbar:
        for doc in tokenized:
            stopword_removed.append([w.strip() for w in doc if w not in stoplist])
            pbar.update(1)

    counter_before = Counter(itertools.chain(*tokenized))
    counter_after = Counter(itertools.chain(*stopword_removed))
    return counter_before, counter_after

def update_stoplist(counter_before, counter_after, topn):
    words_before = [w for w, c in sorted(counter_before.items(), key=lambda x:x[1], reverse=True)]
    words_after = [w for w, c in sorted(counter_after.items(), key=lambda x:x[1], reverse=True)]

    print('========================================')
    for idx, word in enumerate(words_before[:topn]):
        if word in words_after:
            print('  | [{:3,}] BEFORE: {} -> AFTER: {}({:,})'.format((idx+1), word, word, counter_after[word]))
        else:
            print('  | [{:3,}] BEFORE: {}({:,}) -> '.format((idx+1), word, counter_before[word]))
    print('========================================')

if __name__ == '__main__':
    ## Data import
    docs = data_import()

    ## Update stopword list
    fname_tokenized = 'tokenized_corpus_20210715.pk'
    fname_stoplist = 'stoplist_20210715.txt'
    do_tokenize = True
    counter_before, counter_after = count_words(docs=docs, fname_tokenized=fname_tokenized, do_tokenize=do_tokenize, fname_stoplist=fname_stoplist)

    topn = 300
    update_stoplist(counter_before=counter_before, counter_after=counter_after, topn=topn)


import streamlit as st
import pandas as pd
import matplotlib.pyplot as plt

day_df = pd.read_csv('https://raw.githubusercontent.com/hanakorui/Proyek-Akhir-Dicoding/master/Dashboard/Bike-sharing-dataset.%20csv/day.csv')
hour_df = pd.read_csv('https://raw.githubusercontent.com/hanakorui/Proyek-Akhir-Dicoding/master/Dashboard/Bike-sharing-dataset.%20csv/hour.csv')

st.title("Bike Sharing Analysis :bike: :sparkles: ")

average_registered = day_df["registered"].mean()
average_temp = day_df["temp"].mean()

st.sidebar.title("Bike Sharing Analysis :bike:")
st.sidebar.write("Depends on")
side = st.sidebar.selectbox(
    label="Select data", options=("Day", "Hour"))

if side == "Day":
    st.subheader("What weather conditions lead to an increase in daily rental?")

    col1, col2 = st.columns(2)
    with col1:
        st.metric("Average Registered:", average_registered)
    with col2:
        st.metric("Average Temp:", average_temp)

    temp = day_df["temp"]
    register = day_df["registered"]

    fig_day, ax_day = plt.subplots(figsize=(10, 6))
    ax_day.scatter(temp, register, color="blue")
    ax_day.set_title("How Temperature Affects the Number of Registrants")
    ax_day.set_xlabel("Temp (Celcius)")
    ax_day.set_ylabel("Registered")
    ax_day.grid(color="gray")
    st.write(fig_day)

elif side == "Hour":
    st.subheader("How many registered bicycle users are there every hour under increasing weather conditions?")

    std_reg = hour_df["registered"].max()
    std_temp = hour_df["temp"].mean()
    avrg_hr = hour_df["hr"].max()

    col1, col2, col3 = st.columns(3)
    with col1:
        st.metric("Most Registered: ", std_reg)
    with col2:
        st.metric("Hour: ", avrg_hr)
    with col3:
        st.metric("Temp: ", std_temp)

    temp = hour_df["temp"]
    register = hour_df["registered"]

    fig_hour, ax_hour = plt.subplots(figsize=(20, 10))
    ax_hour.scatter(temp, register, color="purple")
    ax_hour.set_title("How Temperature Affects the Number of Registrants")
    ax_hour.set_xlabel("Temp (Celcius)")
    ax_hour.set_ylabel("Registered")
    ax_hour.grid(color="gray")
    st.write(fig_hour)

st.caption("ratihranupadma")


import json

from asgiref.sync import async_to_sync
from channels.generic.websocket import WebsocketConsumer

from .game import DiceGame

class GameConsumer(WebsocketConsumer):
    player_list = []
    rooms = {}

    def connect(self):
        self.room_name = self.scope["url_route"]["kwargs"]["room_name"]
        self.room_group_name = "room_%s" % self.room_name

        async_to_sync(self.channel_layer.group_add)(  # type:ignore
            self.room_group_name,
            self.channel_name
        )

        self.accept()

    def disconnect(self, _):
        GameConsumer.player_list.clear()
        GameConsumer.rooms.clear()

        async_to_sync(self.channel_layer.group_discard)(  # type:ignore
            self.room_group_name,
            self.channel_name
        )

    def checkPlayerStatus(self):
        length = len(GameConsumer.player_list)
        return False if length < 2 else True

    def receive(self, text_data):
        print(text_data)
        data = json.loads(text_data)
        action = data.get("action", "")
        status = action

        if action == "turn":
            if self.checkPlayerStatus():
                req_id = data.get("id", "")
                if self.nextTurn(req_id) is False:
                    return
            else:
                self.sendWaiting()
                return
        elif action == "lock":
            if self.checkPlayerStatus():
                req_id = data.get("id", "")
                lockList = data.get("lockList", "")
                if self.lockDice(req_id, lockList) is False:
                    return
            else:
                self.sendWaiting()
                return
        elif action == "magni":
            if self.checkPlayerStatus():
                if self.updateMagni(data.get("num", "")) is False:
                    return
            else:
                self.sendWaiting()
                return
        elif action == "cal_score":
            if self.checkPlayerStatus():
                self.calScore()
            else:
                self.sendWaiting()
                return
        elif action == "get_id":
            self.getId()
            return
        else:
            return

        async_to_sync(self.channel_layer.group_send)(  # type:ignore
            self.room_group_name,
            {
                "type": "sendMsg",
                "s": status,
            }
        )

    def sendWaiting(self):
        self.send(
            text_data=json.dumps({"status": "waiting"})
        )

    def getId(self):
        players = GameConsumer.player_list
        length = len(players)
        if length == 2:  # 已有两个玩家
            return

        players.append(str(length))
        length += 1

        if length >= 2:
            game = DiceGame(id1=players[0], id2=players[1], first=players[0])
            GameConsumer.rooms[self.room_name] = game

        self.send(
            text_data=json.dumps(
                {"status": "get_id", "id": GameConsumer.player_list[-1]}
            )
        )

    def calScore(self):
        self.game_driver = GameConsumer.rooms[self.room_name]
        self.game_driver.calScore()

    def updateMagni(self, num):
        self.game_driver = GameConsumer.rooms[self.room_name]
        turn = self.game_driver.turn

        k = num
        self.game_driver.incMagnification(k=int(k))
        self.game_driver.turn = int(not turn)
        return True

    def lockDice(self, req_id, lockList):
        self.game_driver = GameConsumer.rooms[self.room_name]
        turn = self.game_driver.turn
        id = self.game_driver.ord2id(turn)
        if id != req_id:
            return False

        if self.game_driver.curRound < self.game_driver.rounds:
            self.game_driver.players[id].lockDice(lockList)
        else:
            self.game_driver.players[id].lockDice([0, 1, 2, 3, 4])
        self.game_driver.turn = int(not turn)
        return True

    def nextTurn(self, req_id):
        self.game_driver = GameConsumer.rooms[self.room_name]
        turn = self.game_driver.turn
        id = self.game_driver.ord2id(turn)
        if id != req_id:
            return False

        k = self.game_driver.players[id].calAvail()
        self.game_driver.rollDice(k)
        self.game_driver.players[id].updateDiceStatus(
            self.game_driver.rollRes
        )

        if turn == int(self.game_driver.first):
            if self.game_driver.curRound < self.game_driver.rounds:
                self.game_driver.curRound += 1
            else:
                self.game_driver.curRound = 0
                self.game_driver.curInning += 1

        if self.game_driver.curRound == self.game_driver.rounds:
            self.game_driver.players[id].lockDice([0, 1, 2, 3, 4])
            self.game_driver.turn = int(not turn)

        return True

    def sendMsg(self, event):
        s = event["s"]
        self.game_driver = GameConsumer.rooms[self.room_name]
        obj = self.game_driver.getStatus()
        obj["status"] = s
        self.send(text_data=json.dumps(obj))


import numpy as np
import cv2
from matplotlib import pyplot as plt
from PIL import Image
import base64
import io

# Match contours to license plate or character template

def find_contours(dimensions, img):
    # Find all contours in the image
    cntrs, _ = cv2.findContours(img.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    # Retrieve potential dimensions
    lower_width = dimensions[0]
    upper_width = dimensions[1]
    lower_height = dimensions[2]
    upper_height = dimensions[3]

    # Check largest 5 or  15 contours for license plate or character respectively
    cntrs = sorted(cntrs, key=cv2.contourArea, reverse=True)[:15]

    ii = cv2.imread('contour.jpg')

    x_cntr_list = []
    target_contours = []
    img_res = []
    for cntr in cntrs:
        # detects contour in binary image and returns the coordinates of rectangle enclosing it
        intx, inty, intwidth, intheight = cv2.boundingRect(cntr)

        # checking the dimensions of the contour to filter out the characters by contour's size
        if lower_width < intwidth < upper_width and lower_height < intheight < upper_height:
            x_cntr_list.append(
                intx)  # stores the x coordinate of the character's contour, to used later for indexing the contours

            char_copy = np.zeros((44, 24))
            # extracting each character using the enclosing rectangle's coordinates.
            character = img[inty:inty + intheight, intx:intx + intwidth]
            character = cv2.resize(character, (20, 40))

            cv2.rectangle(ii, (intx, inty), (intwidth + intx, inty + intheight), (50, 21, 200), 2)
            plt.imshow(ii, cmap='gray')

            #  Make result formatted for classification: invert colors
            character = cv2.subtract(255, character)

            # Resize the image to 24x44 with black border
            char_copy[2:42, 2:22] = character
            char_copy[0:2, :] = 0
            char_copy[:, 0:2] = 0
            char_copy[42:44, :] = 0
            char_copy[:, 22:24] = 0

            img_res.append(char_copy)  # List that stores the character's binary image (unsorted)

    # Return characters on ascending order with respect to the x-coordinate (most-left character first)

    plt.show()
    # arbitrary function that stores sorted list of character indices
    indices = sorted(range(len(x_cntr_list)), key=lambda k: x_cntr_list[k])
    img_res_copy = []
    for idx in indices:
        img_res_copy.append(img_res[idx])  # stores character images according to their index
    img_res = np.array(img_res_copy)

    return img_res

def extract_plate(image):
    plate_img = image.copy()

    plate_cascade = cv2.CascadeClassifier('C:\\indian_license_plate.xml')

    # detects number plates and returns the coordinates and dimensions of detected license plate's contours

    plate_rect = plate_cascade.detectMultiScale(plate_img, scaleFactor=1.3, minNeighbors=7)

    for (x, y, w, h) in plate_rect:
        a, b = (int(0.02 * img.shape[0]), int(0.025 * img.shape[1]))  # parameter tuning
        plate = plate_img[y + a:y + h - a, x + b:x + w - b, :]
        # finally representing the detected contours by drawing rectangles around the edges.
        cv2.rectangle(plate_img, (x, y), (x + w, y + h), (51, 51, 255), 3)
    # plt.im show(plate, c map='gray')
    # plt.show()
    # cv2.im write('plate.jpg',plate)

    return plate  # returning the processed image

# Find characters in the resulting images

def segment_characters(image):
    image = extract_plate(image)
    # Preprocess cropped license plate image
    img_lp = cv2.resize(image, (333, 75))
    img_gray_lp = cv2.cvtColor(img_lp, cv2.COLOR_BGR2GRAY)
    _, img_binary_lp = cv2.threshold(img_gray_lp, 200, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
    img_binary_lp = cv2.erode(img_binary_lp, (3, 3))
    img_binary_lp = cv2.dilate(img_binary_lp, (3, 3))

    lp_width = img_binary_lp.shape[0]
    lp_height = img_binary_lp.shape[1]

    # Make borders white
    img_binary_lp[0:3, :] = 255
    img_binary_lp[:, 0:3] = 255
    img_binary_lp[72:75, :] = 255
    img_binary_lp[:, 330:333] = 255

    # Estimations of character contours sizes of cropped license plates
    dimensions = [lp_width / 6, lp_width / 2, lp_height / 10, 2 * lp_height / 3]
    plt.imshow(img_binary_lp, cmap='gray')
    plt.show()
    cv2.imwrite('contour.jpg', img_binary_lp)

    # Get contours within cropped license plate
    char_list = find_contours(dimensions, img_binary_lp)

    return char_list

def main(data):
    decoded_data = base64.b64decode(data)
    np_data = np.fromstring(decoded_data, np.uint8)
    img = cv2.imdecode(np_data, cv2.IMREAD_UNCHANGED)
    pil_im = Image.fromarray(segment_characters(img))

    buff = io.BytesIO
    pil_im.save = (buff, format("JPEG"))
    img_str = base64.b64encode(buff.getvalue())
    return""+str(img_str, "utf-8")


counter = 0
cache = {}

# def F(n):
#     global counter
#     counter += 1
#     if n >= 1:
#         counter += 1
#         F(n - 1)
#         F(n - 3)
#         counter += 1

def F(n):
    global cache
    if n in cache:
        return cache[n]
    if n >= 1:
        cache[n] = F(n - 1) + F(n - 3) + 2 + 1
        return cache[n]
    else:
        cache[n] = 1
        return cache[n]

print(F(500))
print(cache)


# What is the largest prime factor of the number 600851475143 ?
# Answer: 6857

primes = [2]
z = 600851475143
i = 1
k = 2
while i < z / k:
    if i > primes[len(primes)-1]:
        isPrime = True
        for p in primes:
            if i % p == 0:
                isPrime = False
        if isPrime == True:
            primes.append(i)
            k = i
            if z % i == 0:
                z /= i
                i = 1
    else:
        for p in primes:
            if i == p:
                if z % i == 0:
                    z /= i
                    i = 1
    i += 1
print(z)

# The objective is to strip prime factors from z in order to find the largest irreducible factor of z as quickly as we can.
# The idea behind this is to teach the program how to identify and keep track of prime numbers as we iterate upwards.
# z is large and to keep it from taking too much time, we want to only deal with small iterations.
# Therefore the strategy is to strip away the lowest prime factors we're able to find until we're left with the largest prime factor -
# for which any i, it's unable to divide fully.
# We split it up into two forms to minimize time taken: whether we need to treat it as potential addition to our prime list or whether
# to identify whether or not i is already listed and whether it's able to be stripped from z.

# Calculation time (on my PC): 15619 Microseconds


from loguru import logger

from sqlalchemy import text
from sqlalchemy.engine import Engine

def drop_recreate_database(engine: Engine, db_name: str) -> None:
    with engine.connect() as connection:
        drop_db = text(f"DROP DATABASE IF EXISTS {db_name}")
        connection.execute(drop_db)
        create_db = text(f"CREATE DATABASE {db_name}")
        connection.execute(create_db)
        logger.warning("Dropped and recreated database.")


from angular_servo import AngularServo
from async_runner import add_task, start_tasks
from async_tasks import cpu_load, heartbeat

# HARDWARE SETUP:
# Materials:
#            servo motor
#
# connect the servo motor:
#                    connect to the power and GND
#                    connect signal to pin 4

def engine_run():
    # The speed pin is used by the stepper motor driver to time when to take
    # a step. The smaller the delay, the quicker the RPM. If this is made too
    # small, the engine will not move at all.

    servo = AngularServo(control_pin=10)

    while True:
        for angle in [0, 45, 90, 45, 0, -45, -90, -45]:
            servo.angle = angle  # set the servo to the new angle
            yield 1000  # wait 1s nfor the servo to move to the new position

add_task(cpu_load)
add_task(heartbeat)
add_task(engine_run)

start_tasks()


# Bibliotecas
import pandas as pd
import datetime
import plotly.express as px
import folium
from haversine import haversine
import streamlit as st
from PIL import Image
from streamlit_folium import folium_static

# =============================================================
# Funções
# =============================================================

def carrega_dataframe():
    """ Função para carregar o dataframe para a memória

    Returns:
        dataframe: retorna dataframe com a base carregada
    """
    path = 'dados/'
    df = pd.read_csv(path + 'train.csv')
    df1 = df.copy()
    return df1

def tratamento_dataframe(df1):
    """Função para fazer o tratamento e limpeza da base de dados
    1. Retira espaços em branco das variáveis de texto
    2. Retira dados faltantes
    3. Ajusta formato das variáveis
    4. Retira texto da variável de tempo (numérica)
    5. Cria variável de semana do ano
    6. Cria variável de distância da entrega

    Args:
        df1 (dataframe): leitura do dataframe carregado na memória

    Returns:
        dataframe: retorna dataframe com todas as limpezas e tratamentos
    """

    # Retirando espaços em branco das variáveis categóricas
    df1.loc[:, 'ID'] = df1.loc[:, 'ID'].str.strip()
    df1.loc[:, 'Delivery_person_ID'] = df1.loc[:,
                                               'Delivery_person_ID'].str.strip()
    df1.loc[:, 'Road_traffic_density'] = df1.loc[:,
                                                 'Road_traffic_density'].str.strip()
    df1.loc[:, 'Type_of_order'] = df1.loc[:, 'Type_of_order'].str.strip()
    df1.loc[:, 'Type_of_vehicle'] = df1.loc[:, 'Type_of_vehicle'].str.strip()
    df1.loc[:, 'Festival'] = df1.loc[:, 'Festival'].str.strip()
    df1.loc[:, 'City'] = df1.loc[:, 'City'].str.strip()

    # Retirando dados faltantes
    df1 = df1[(df1['Delivery_person_Age'] != 'NaN ')
              & (df1['multiple_deliveries'] != 'NaN ')
              & (df1['Road_traffic_density'] != 'NaN')
              & (df1['City'] != 'NaN')
              & (df1['Weatherconditions'] != 'conditions NaN')
              & (df1['Festival'] != 'NaN')
              ].reset_index(drop=True)

    # Ajustando o formato das variáveis
    df1['Delivery_person_Age'] = df1['Delivery_person_Age'].astype('int64')
    df1['Delivery_person_Ratings'] = df1['Delivery_person_Ratings'].astype(
        float)
    df1['Order_Date'] = pd.to_datetime(df1['Order_Date'], format='%d-%m-%Y')
    df1['multiple_deliveries'] = df1['multiple_deliveries'].astype('int64')

    # Retirando texto (min) da coluna de tempo de entrega
    df1['Time_taken(min)'] = df1['Time_taken(min)'].apply(
        lambda x: x.split('(min) ')[1])
    df1['Time_taken(min)'] = df1['Time_taken(min)'].astype('int64')

    # Criando variável do dia da semana
    df1['week_of_year'] = df1['Order_Date'].dt.strftime('%U')
    df1['week_of_year'] = df1['week_of_year'].astype('int64')

    # Criando variavel de distancia
    df1['distancia'] = df1.apply(lambda x: haversine((x['Restaurant_latitude'], x['Restaurant_longitude']),
                                                     (x['Delivery_location_latitude'], x['Delivery_location_longitude'])), axis=1)

    df1 = df1.reset_index()
    df2 = df1.copy()
    return df2

df1 = carrega_dataframe()
df2 = tratamento_dataframe(df1)

# =======================================================
#  STREAMLIT
# =======================================================

# =======================================================
#  CONFIGURA PÁGINA
# =======================================================
st.set_page_config(page_title='Visão Empresa', page_icon='📈', layout="wide",
                   initial_sidebar_state="auto", menu_items=None)
st.markdown('# Cury Company - **Visão Empresa**')

# =======================================================
#  BARRA LATERAL
# =======================================================
path = ''
image = Image.open(path + 'img.jpg')
st.sidebar.image(image, width=300)
st.sidebar.markdown('# **Cury Company Delivery**🍕')

st.sidebar.markdown('---')
st.sidebar.markdown('### Selecione uma data limite')
date_slider = st.sidebar.slider('',
                                # value=datetime.datetime(2022, 4, 13),
                                value=datetime.datetime(df2['Order_Date'].max(
                                ).year, df2['Order_Date'].max().month, df2['Order_Date'].max().day),
                                min_value=datetime.datetime(df2['Order_Date'].min(
                                ).year, df2['Order_Date'].min().month, df2['Order_Date'].min().day),
                                max_value=datetime.datetime(df2['Order_Date'].max(
                                ).year, df2['Order_Date'].max().month, df2['Order_Date'].max().day),
                                format='DD-MM-YYYY')

st.sidebar.markdown('---')
st.sidebar.markdown('### Selecione as condições de trânsito')
traffic_options = st.sidebar.multiselect('',
                                         df2['Road_traffic_density'].unique(),
                                         default=df2['Road_traffic_density'].unique())

st.sidebar.markdown('---')
st.sidebar.markdown('### Selecione o tipo de veículo')
vehicle_options = st.sidebar.multiselect('',
                                         df2['Type_of_vehicle'].unique(),
                                         default=df2['Type_of_vehicle'].unique())

st.sidebar.markdown('---')
st.sidebar.markdown('### Selecione o tipo de pedido')
order_options = st.sidebar.multiselect('',
                                       df2['Type_of_order'].unique(),
                                       default=df2['Type_of_order'].unique())

st.sidebar.markdown('---')
st.sidebar.markdown('### Selecione o tipo de cidade')
city_options = st.sidebar.multiselect('',
                                      df2['City'].unique(),
                                      default=df2['City'].unique())

st.sidebar.markdown('---')
st.sidebar.markdown('### Selecione as condições climáticas')
weather_options = st.sidebar.multiselect('',
                                         df2['Weatherconditions'].unique(),
                                         default=df2['Weatherconditions'].unique())

st.sidebar.markdown('---')
st.sidebar.markdown('')
st.sidebar.markdown('##### Criado por **Adérito Bernardes**')
st.sidebar.markdown('---')

# =======================================================
#  APLICANDO FILTROS
# =======================================================

df2 = df2.loc[df2['Order_Date'] <= date_slider, :]
df2 = df2.loc[df2['Road_traffic_density'].isin(traffic_options), :]
df2 = df2.loc[df2['Type_of_vehicle'].isin(vehicle_options), :]
df2 = df2.loc[df2['Type_of_order'].isin(order_options), :]
df2 = df2.loc[df2['City'].isin(city_options), :]
df2 = df2.loc[df2['Weatherconditions'].isin(weather_options), :]

# =======================================================
#  LAYOUT STREAMLIT
# =======================================================
tab1, tab2, tab3 = st.tabs(
    ['Visão Gerencial', 'Visão Estratégica', 'Visão Geográfica'])

with tab1:
    with st.container():
        st.markdown('##### Pedidos por dia')
        df_aux = df2[['Order_Date', 'ID']].groupby(
            ['Order_Date']).count().reset_index()
        df_aux.columns = ['Order_Date', 'qtd_entregas']
        fig = px.bar(df_aux, x='Order_Date', y='qtd_entregas', labels={'Order_Date': 'Data do pedido',
                                                                       'qtd_entregas': 'Qtd entregas'
                                                                       })
        st.plotly_chart(fig, use_container_width=True)

    with st.container():
        col1, col2 = st.columns(2, gap='large')
        with col1:
            st.markdown('##### Pedidos por densidade de tráfego')
            df_aux = df2[['Road_traffic_density', 'ID']].groupby(
                ['Road_traffic_density']).count().reset_index()
            df_aux.columns = ['Road_traffic_density', 'qtd_entregas']
            df_aux['perc_ID'] = 100 * \
                (df_aux['qtd_entregas']/df_aux['qtd_entregas'].sum())
            fig = px.pie(df_aux, values='perc_ID', names='Road_traffic_density', labels={'Road_traffic_density': 'Densidade de tráfego',
                                                                                'perc_ID': '% entregas'
                                                                                })
            st.plotly_chart(fig, use_container_width=True)
        with col2:
            st.markdown('##### Pedidos por densidade de tráfego e cidade')
            df_aux = df2[['Road_traffic_density', 'City', 'ID']].groupby(
                ['City', 'Road_traffic_density']).count().reset_index()
            df_aux.columns = ['Road_traffic_density', 'City', 'qtd_entregas']
            df_aux['perc_ID'] = 100 * \
                (df_aux['qtd_entregas']/df_aux['qtd_entregas'].sum())

            fig = px.bar(df_aux, x='City', y='qtd_entregas', color='Road_traffic_density', barmode='group', labels={'Road_traffic_density': 'Densidade de tráfego',
                                                                                'City': 'Cidade',
                                                                                'qtd_entregas': 'Qtd entregas'})
            st.plotly_chart(fig, use_container_width=True)

with tab2:
    with st.container():
        st.markdown('##### Pedidos por semana')
        df_aux = df2[['week_of_year', 'ID']].groupby(
            ['week_of_year']).count().reset_index()
        df_aux.columns = ['week_of_year', 'qtd_entregas']
        fig = px.line(df_aux, x='week_of_year', y='qtd_entregas', labels={'week_of_year': 'Semana do ano',
                                                                          'qtd_entregas': 'Qtd entregas'
                                                                          })
        st.plotly_chart(fig, use_container_width=True)

    with st.container():
        st.markdown('##### Pedidos por entregador por semana')
        df_aux1 = df2.loc[:, ['ID', 'week_of_year']].groupby(
            'week_of_year').count().reset_index()
        df_aux2 = df2.loc[:, ['Delivery_person_ID', 'week_of_year']].groupby(
            'week_of_year').nunique().reset_index()
        df_aux = pd.merge(df_aux1, df_aux2, how='inner')
        df_aux['order_by_delivery'] = df_aux['ID'] / \
            df_aux['Delivery_person_ID']
        fig = px.line(df_aux, x='week_of_year', y='order_by_delivery', labels={'week_of_year': 'Semana do ano',
                                                                               'order_by_delivery': 'Pedidos por entregador'})
        st.plotly_chart(fig, use_container_width=True)

with tab3:
    st.markdown('##### Mapa geográfico das entregas')
    columns = [
        'City',
        'Road_traffic_density',
        'Delivery_location_latitude',
        'Delivery_location_longitude'
    ]

    columns_grouped = ['City', 'Road_traffic_density']
    data_plot = df2.loc[:, columns].groupby(
        columns_grouped).median().reset_index()
    data_plot.columns = [
        'Cidade', 'Densidade de tráfego', 'latitude', 'longitude']

    # Desenhar o mapa
    map_ = folium.Map(zoom_start=11)
    for index, location_info in data_plot.iterrows():
        folium.Marker([location_info['latitude'],
                       location_info['longitude']],
                      popup=location_info[['Cidade', 'Densidade de tráfego']],
                      icon=folium.Icon(color='red', icon='info-sign')).add_to(map_)
    folium_static(map_, width=1024, height=500)