"""
Adapter for areas
"""

import httplib
import logging
import json
import base64
import re
from decimal import Decimal

from datetime import datetime

from django.contrib.auth.models import User
from django.contrib.gis.geos import MultiPolygon
from django.contrib.gis.geos import Polygon
from django.template.defaultfilters import slugify
from django.core.exceptions import ValidationError
from django.db.models.fields import DateField

from lizard_area.models import Area
from lizard_area.models import AreaWFSConfiguration
from lizard_area.models import SynchronizationHistory

from lizard_geo.models import GeoObjectGroup

from lizard_measure.models import KRWWatertype
from lizard_measure.models import WaterBody

FIELDS_MAPPING_PEILGEBIED = {
    'gpgident': 'ident',
    'gpgnaam': 'name',
    'iws_dtmwijzmeta': 'dt_latestchanged_krw',
    'gpgoppvl': 'surface',
    'gpgsoort': 'areasort',
    'gpgsoort_krw': 'areasort_krw'
}

FIELDS_MAPPING_AANAFVOERGEBIED = {
    'gafident': 'ident',
    'gafnaam': 'name',
    'iws_dtmwijzmeta': 'dt_latestchanged_krw',
    'gafoppvl': 'surface',
    'gafsoort': 'areasort',
    'gafsoort_krw': 'areasort_krw'
}

FIELDS_MAPPING_EXCEPTION = {
    'fsoort_krw': 'gafsoort_krw',
    'wwatertype': 'krwwatertype',
    'angafident': 'onderdeelvangafident',
    'angpgident': 'onderdeelvangpgident',
    'tailniveau': 'detailniveau'
}

default_logger = logging.getLogger(__name__)

class Synchronizer(object):
    """Synchronize 'aanafvoergebieden' with 'geovoorzieningen',
    build a parent-child relation for synchronized objecten,
    create waterbodies."""

    def __init__(self, logger=None):
        self.logger = logger
        if self.logger is None:
            self.logger = default_logger

    def log_synchistory(self, sync_hist, **kwargs):
        """Save synchronistaion history.

        Arguments:

        sync_hist -- instance of SynchronizationHistory
        kwargs - dict contains a field name end value
        """
        for k, v in kwargs.items():
            setattr(sync_hist, k, v)
        self.logger.info(
            ' | '.join(["%s=%s" % (k, v) for k, v in kwargs.items()]))
        sync_hist.save()

    def replace_suffix(self, date_string):
        """Replace suffix at the end of the string.
        This is because the bug in geoserver at date fields.

        example: replace 'Z' from '11-11-2012Z'"""
        if not isinstance(date_string, (str, unicode)):
            return date_string
        return re.split('[a-zA-Z]$', date_string)[0]

    def geo_object_group(self):
        """Return an instance of GeoObjectGroup."""
        group_name = 'LAYERS'
        group_slug = slugify(group_name)
        geo_object_group_user = User.objects.filter(is_superuser=True)[0]
        geo_object_group, created = GeoObjectGroup.objects.get_or_create(
            name=group_name,
            slug=group_slug,
            defaults={'created_by': geo_object_group_user},
        )
        if created:
            geo_object_group.source_log = 'LAYERS'
            geo_object_group.save()
        return geo_object_group

    def fields_mapping(self, area_type):
        """Maps GV fields with Area where
        key is GV field, value is Area field.

        Arguments:
        area_type - 'peilgebied' or 'aanafvoergebied'.
        """
        if area_type.lower() == 'peilgebied':
            return FIELDS_MAPPING_PEILGEBIED
        elif area_type.lower() == 'aanafvoergebied':
            return FIELDS_MAPPING_AANAFVOERGEBIED
        else:
            return {}

    def geometry2mp(self, geo_dict):
        """Convert dict to MultiPolygon object.

        Arguments:
        geo_dict -- python dict with in 'Multipoligon' as type
        and nested list of points as 'coordinates'
        """
        coord = geo_dict.get('coordinates', None)
        geo_type = geo_dict.get('type', None)
        if coord is None or geo_type is None:
            self.logger.error("Expected 'type' and 'coordinates' "
                              "keys to convert dict to MultiPolygon "
                              "object, given %s" % geo_dict.keys())
            return None

        if geo_type.lower() == 'multipolygon':
            polygons = []
            for c in coord:
                [polygons.append(Polygon(item)) for item in c]

            if len(polygons) > 0:
                return MultiPolygon(polygons)
            else:
                self.logger.debug(
                    "Amount of polygons is '%d'." % len(polygons))
        return None

    def check_content(self, content):
        """Checks or content contains a GeoJSON elements.

        Arguments:

        content -- dict contains data from wfs
        """

        expected_keys = [u'crs', u'type', u'features', u'bbox']
        features_key = u'features'
        if isinstance(content, dict) is False:
            return False

        for key in expected_keys:
            if key not in content.keys():
                return False

        features = content[features_key]
        if len(features) <= 0:
            return False

        expected_propertie_keys = [u'geometry',
                                   u'properties',
                                   u'type',
                                   u'id',
                                   u'geometry_name']
        for feature in features:
            if isinstance(feature, dict) is False:
                return False
            for key in expected_propertie_keys:
                if key not in feature.keys():
                    return False
            if isinstance(feature['geometry'], dict) is False:
                return False
        return True

    def get_ident(self, properties, area_type):
        """Returns ident of passed area object.
        The ident field has different names depends on area_type.

        Arguments

        properties -- dict contains area data from wfs
        area_type -- area type as string for example 'peilgebied'
        """
        if area_type == 'aanafvoergebied':
            return properties.get('gafident')
        if area_type == 'peilgebied':
            return properties.get('gpgident')
        return None

    def get_parent_area(self, properties):
        """Return parent object of passed area object."""
        gafident_parent = properties.get('onderdeelvangafident', None)
        gpgident_parent = properties.get('onderdeelvangpgident', None)
        parent_areas1 = Area.objects.filter(ident=gafident_parent)
        parent_areas2 = Area.objects.filter(ident=gpgident_parent)

        if parent_areas1.exists():
            return parent_areas1[0]
        elif parent_areas2.exists():
            return parent_areas2[0]
        else:
            return None

    def get_krw_watertype(self, krwwatertype, sync_hist):
        """ Return KRWWatertype object."""
        try:
            watertype = KRWWatertype.objects.get(code=krwwatertype)
            return watertype
        except:
            message = "KrwWaterType '%s' is is not in Aqua Domain table." % (
                krwwatertype)
            self.logger.debug(message)
            return None

    def update_or_create_waterbody(self, area, krwwatertype, sync_hist):
        """Update or create Waterbody object"""
        waterbody, created = WaterBody.objects.get_or_create(area=area)
        krw_watertype = self.get_krw_watertype(krwwatertype, sync_hist)
        if krw_watertype != waterbody.krw_watertype:
            waterbody.krw_watertype = krw_watertype
            waterbody.save()
            return True
        return False

    def update_area(self, area_object, properties, geometry,
                    area_type, username, data_set, sync_hist):
        """Updates a passed area object. Returns
        amount of activated and updated objects.
        Remove suffix from value of date fields due bug in wfs pp418.

        Arguments:

        area_object -- instance object to update
        properties -- dict contains area data
        geometry -- dict contains geometry data
        data_set -- instance object of lizard_security.DataSet
        area_type -- area type as string for example 'peilgebied'
        username -- user logging name as string
        """
        updated = False
        activated = False
        mapping = self.fields_mapping(area_type)
        for k, v in mapping.items():
            if k not in properties.keys():
                self.logger.warning(
                    "Wrong fields mapping, expected field '%s' NOT in response." % k)
                continue
            value_krw = properties.get(k)
            value_vss = getattr(area_object, v)
            field_vss = area_object._meta.get_field(v)
            if isinstance(value_vss, unicode) and isinstance(value_krw, int):
                value_krw = unicode(value_krw)
            if isinstance(value_vss, unicode) and isinstance(value_krw, Decimal):
                value_krw = unicode(value_krw)
            if isinstance(field_vss, DateField) and isinstance(value_krw, unicode):
                value_krw = self.replace_suffix(unicode(value_krw))
                value_vss = unicode(value_vss)
            if isinstance(value_krw, unicode):
                value_krw = value_krw.encode('ascii', 'ignore')
            if value_vss != value_krw:
                setattr(area_object, v, value_krw)
                self.logger.debug("Update {0} with {1} old value {2}".format(
                        v, value_vss, value_krw))
                updated = True

        mp = self.geometry2mp(geometry)

        if getattr(area_object, 'geometry') != mp:
            setattr(area_object, 'geometry', mp)
            self.logger.debug("Update geometry")
            updated = True

        group = self.geo_object_group()
        if getattr(area_object, 'geo_object_group') != group:
            setattr(area_object, 'geo_object_group', group)
            self.logger.debug("Update geo object group")
            updated = True

        detailniveau = properties.get('detailniveau', None)
        if detailniveau is not None or detailniveau != '':
            if getattr(area_object, 'is_active') is False:
                setattr(area_object, 'is_active', True)
                self.logger.debug("Update detailniveau")
                updated = True
                activated = True

        if getattr(area_object, 'area_type') != area_type:
            setattr(area_object, 'area_type', area_type)
            self.logger.debug("Update area type")
            updated = True

        area_class = Area.AREA_CLASS_AAN_AFVOERGEBIED
        if getattr(area_object, 'area_class') != area_class:
            setattr(area_object, 'area_class', area_class)
            self.logger.debug("Update area class")
            updated = True

        if getattr(area_object, 'data_set') != data_set:
            setattr(area_object, 'data_set', data_set)
            self.logger.debug("Update data set")
            updated = True

        krwwatertype = properties.get('krwwatertype', None)
        if self.update_or_create_waterbody(area_object, krwwatertype, sync_hist):
            self.logger.debug("Update waterbody")
            updated = True

        return updated, activated

    def set_extra_values(self, area_object, created, updated):
        """Sets extra values into passed area_object.

        Arguments:

        area_object -- instance object of lizard_area.Area
        created - boolean indicates a new object
        updated - boolean indicates a existing object
        """
        now = datetime.today()
        setattr(area_object, 'dt_latestsynchronized', now)
        if created:
            setattr(area_object, 'dt_created', now)
        if updated:
            setattr(area_object, 'dt_latestchanged', now)

    def set_parent_relation(self, features, area_type):
        """Create a child-parent relation.

        Arguments:

        features -- list of dict contains krw areas from wfs
        area_type -- area type as string, for example 'peilgebied'
        """
        self.logger.info("Create a child-parent relation.")
        for feature in features:
            properties = self.properties_keys_to_lower(feature['properties'])
            ident = self.get_ident(properties, area_type)
            areas = Area.objects.filter(ident=ident)
            if areas.exists():
                area = areas[0]
                area.parent = self.get_parent_area(properties)
                self.logger.debug(
                    "Set parent='%s' to child='%s'." % (area.parent, ident))
                area.save()
            else:
                self.logger.info(
                    'Area with krw_ident=%s does not exist.' % ident)

    def invalidate(self, content, area_type, data_set):
        """Deactivate areas those available in vss
        but not in content. Deactivates areas where
        'detailniveau' is empty.
        Returns amount of deactivated areas.

        Arguments:

        content -- list of dict contains krw areas from wfs
        area_type -- area type as string for example 'peilgebied'
        data_set -- instance object of lizard_security.DataSet
        """
        vss_areas = Area.objects.filter(area_type=area_type,
                                        data_set=data_set,
                                        is_active=True)
        amount_deactivated = 0
        if vss_areas.exists() is False:
            return amount_deactivated

        for area in vss_areas:
            is_active = False
            for feature in content:
                properties = self.properties_keys_to_lower(
                    feature['properties'])
                ident_krw = self.get_ident(properties, area_type)
                detailniveau = properties.get('detailniveau', None)
                if area.ident == ident_krw and detailniveau is not None:
                    is_active = True
                    break

            if is_active is False:
                area.is_active = is_active
                area.save()
                amount_deactivated = amount_deactivated + 1
        return amount_deactivated

    def get_or_create_area(self, geo_object_group, geometry, ident, data_set):
        """Returns area object. This function to catch MultipleObjectsReturned
        error. The error can raise because Ident is not unique field in db.
        In case of error the function returns None."""
        created = False
        area_object = None
        try:
            area_object = Area.objects.get(ident=ident)
            return area_object, created
        except Area.DoesNotExist as ex:
            area_object = Area(ident=ident,
                               geo_object_group=geo_object_group,
                               geometry=geometry,
                               data_set=data_set,
                               area_class=Area.AREA_CLASS_AAN_AFVOERGEBIED)
            created = True
            return area_object, created
        except Exception as ex:
            self.logger.error(".".join(map(str, ex.args)))
        return area_object, created

    def properties_keys_to_lower(self, orig_dict):
        """Set keys to lawer case, map exception keys."""
        special_dict = {}
        for key, value in orig_dict.iteritems():
            lowercase_key = key.lower()
            new_key = FIELDS_MAPPING_EXCEPTION.get(
                lowercase_key, lowercase_key)
            special_dict[new_key] = value

        return special_dict

    def create_update_areas(self, content, username,
                            area_type, data_set, sync_hist):
        """Creates or updates areas.
        During the execution logs into SynchronizationHistory.

        Arguments:

        content -- dict of geojson format within loaded krw data
        username -- user logging name as string
        area_type -- area type as string for example 'peilgebied'
        data_set -- instance object of lizard_security.DataSet
        sync_hist -- object instance of lizard_area.SynchronizationHistory

        """
        amount_updated = 0
        amount_created = 0
        amount_activated = 0
        amount_synchronized = 0
        amount_deactivated = 0

        for feature in content['features']:
            properties = self.properties_keys_to_lower(feature['properties'])
            geometry = feature['geometry']

            geometry_mp = self.geometry2mp(geometry)
            ident = self.get_ident(properties, area_type)
            self.logger.debug("Synchronise %s ident=%s." % (area_type, ident))
            area_object, created = self.get_or_create_area(
                self.geo_object_group(),
                geometry_mp,
                ident,
                data_set)
            if area_object is None:
                continue
            updated, activated = self.update_area(area_object, properties,
                                                  geometry, area_type,
                                                  username, data_set,
                                                  sync_hist)
            self.set_extra_values(area_object, created, updated)

            if created or updated:
                try:
                    area_object.full_clean()
                    area_object.save()
                    if created:
                        amount_created = amount_created + 1
                    elif updated:
                        amount_updated = amount_updated + 1

                    if activated:
                        amount_activated = amount_activated + 1
                except ValidationError as ex:
                    msg = ' '.join(
                        ['FIELD:' + k + ', VALUE:' + area_object.__dict__[k] + ', MESSAGE: ' + '. '.join(v)
                         for k, v in ex.message_dict.iteritems()])
                    self.logger.error("Error on area.save() : {0}".format(msg))
                except Exception as ex:
                    self.logger.error('Object ident="%s" is not saved' % ident)

        amount_deactivated = self.invalidate(
            content['features'], area_type, data_set)
        amount_synchronized = len(content['features'])
        self.log_synchistory(sync_hist,
                             **{'amount_created': amount_created,
                                'amount_updated': amount_updated,
                                'amount_activated': amount_activated,
                                'amount_synchronized': amount_synchronized,
                                'amount_deactivated': amount_deactivated})

    def get_content(self, response):
        """Return decoded JSON object or string.

        Argument:
        response -- responce object of HTTP request
        """
        try:
            content = json.loads(response.read())
            return content, True
        except ValueError:
            return response.read(), False

    def sync_areas(self, username, url, area_type, data_set, sync_hist, kw):
        """Create a http request, loads the data as json,
        checks or recived data geojson elements, synchronizes the data.
        During the execution logs into SynchronizationHistory.

        Arguments:

        username -- user logging name as string
        params_str -- request parameter as url string
        area_type -- area type as string for example 'peilgebied'
        data_set -- instance object of lizard_security.DataSet
        sync_hist -- object instance of lizard_area.SynchronizationHistory
        kw -- connection parameters as dict containing hostname and login data
        """
        host = kw["host"]
        kwargs = {"url": url, "host": host, "message": "Connecting.."}
        self.log_synchistory(sync_hist, **kwargs)
        connection = httplib.HTTPConnection(host)
        if kw.get('auth') is not None:
            headers = {"Authorization": kw.get('auth'),
                       "Content-type": "application/json",
                       "Accept": "text/plain"}
            connection.request("GET", url, None, headers)
        else:
            connection.request("GET", url)
        response = connection.getresponse()
        success = False
        if response.status == 200:
            message = "Connected, starting load data."
            self.log_synchistory(sync_hist, **{'message': message})
            content, is_json = self.get_content(response)
            if is_json and self.check_content(content):
                message = "Data loaded, synchronization in progress."
                self.log_synchistory(sync_hist, **{'message': message})
                self.logger.debug(message)
                self.create_update_areas(content, username,
                                         area_type, data_set,
                                         sync_hist)
                self.set_parent_relation(content['features'], area_type)
                success = True
            else:
                message = "Content is not a GeoJSON format or empty."
                self.log_synchistory(sync_hist, **{'message': message})
                self.logger.error(message)
        elif response.status == 404:
            message = "Page Not Found host='%s' url='%s'" % (host, url)
            self.log_synchistory(sync_hist, **{'message': message})
            self.logger.error(message)
        else:
            message = "Connection error status='%s' reason='%s'" % (
                response.status, response.reason)
            self.log_synchistory(sync_hist, **{'message': message})
            self.logger.error(message)
        connection.close()
        return success

    def default_request_parameters(self):
        """Containts request parameters. Expected that this parameter
        will not be changed."""
        parameters = {
            'service': 'WFS',
            'version': '1.0.0',
            'request': 'GetFeature',
            'outputFormat': 'json',
            'srsName': 'EPSG:4326'}
        return parameters

    def set_request_parameters(self, config):
        request_parameters = self.default_request_parameters()
        if config.typeName is not None and config.typeName != '':
            request_parameters['typeName'] = config.typeName
        if config.cql_filter is not None and config.cql_filter != '':
            request_parameters['cql_filter'] = config.cql_filter
        if config.maxFeatures is not None:
            request_parameters['maxFeatures'] = config.maxFeatures
        return request_parameters

    def set_url(self, request_parameters, config):
        url = None
        if config.path is None:
            return url
        if config.url is None or config.url == "":
            params_string = '&'.join(
                ['%s=%s' % (k, v) for k, v in request_parameters.items()])
            return "%s?%s" % (config.path, params_string)
        else:
            return "%s?%s" % (config.path, config.url)

    def set_connection_parameters(self, config):
        connection_parameters = {}
        if config.host is None:
            return None
        connection_parameters['host'] = config.host
        if config.username == '':
            return connection_parameters
        if config.username is None:
            return connection_parameters
        pwd = "{0}:{1}".format(config.username, config.password)
        auth = 'Basic ' + str.strip(base64.encodestring(pwd))
        connection_parameters['auth'] = auth
        return connection_parameters

    def run_sync(self, username, area_type, data_set):
        """Retrieves the request configuration,
        creates a request string and starts syncronization.
        During the execution logs into SynchronizationHistory.

        Arguments:

        username -- user logging name as string
        area_type -- area type as string for example 'peilgebied'
        data_set -- instance object of lizard_security.DataSet
        """
        configurations = AreaWFSConfiguration.objects.filter(
            area_type=area_type, data_set=data_set)

        message = "Syncronization is started. Retrieving configuration...."
        success = True
        sync_hist = SynchronizationHistory(
            dt_start=datetime.today(),
            username=username,
            message=message,
            data_set=data_set)
        sync_hist.save()
        if configurations.exists():
            for config in configurations:
                url = self.set_url(self.set_request_parameters(config), config)
                connection_parameters = self.set_connection_parameters(config)
                if connection_parameters is None or url is None:
                    self.logger.warning(
                        'WFS is not properly configured for "%s".' % (
                            config.name))
                    continue
                success = self.sync_areas(
                    username, url, area_type,
                    data_set, sync_hist, connection_parameters)
        else:
            message = "There are no any configuration for %s of %s" % (
                area_type, data_set.name)
            self.log_synchistory(sync_hist, **{'message': message})
            self.logger.info(message)
        if success:
            message = "Synchronization is finished."
        else:
            message = "Synchronization is finished with erors: %s" % (
                sync_hist.message)
        self.log_synchistory(
            sync_hist, **{'dt_finish': datetime.today(), 'message': message})


#!/usr/bin/env python3
# vim: set fileencoding=utf8 :

import types
from contextlib import contextmanager
from tornado import gen
from tornado.concurrent import Future, is_future
import inflection
import python_jsonschema_objects as pjs

from tornado_jsonapi.exceptions import MissingResourceSchemaError

class Resource:
    class ResourceObject:
        def id_(self):
            raise NotImplementedError

        def type_(self):
            raise NotImplementedError

        def attributes(self):
            raise NotImplementedError

        # TODO : relationships, links, meta

    def __init__(self, schema):
        self.schema = schema
        builder = pjs.ObjectBuilder(self.schema)
        classes = builder.build_classes()
        name = inflection.camelize(self.name())
        if name not in classes:
            raise MissingResourceSchemaError(name)
        self._schema = classes[name]

    def _on_request_end(self):
        pass

    def name(self):
        raise NotImplementedError

    def exists(self, id_):
        return self.read(id_) is not None

    def create(self, attributes):
        raise NotImplementedError

    def read(self, id_):
        raise NotImplementedError

    def update(self, id_, attributes):
        raise NotImplementedError

    def delete(self, id_):
        raise NotImplementedError

    def list_(self, limit=0, page=0):
        raise NotImplementedError

    def list_count(self):
        raise NotImplementedError

try:
    import sqlalchemy
    import alchemyjsonschema
    import alchemyjsonschema.dictify
except ImportError:
    pass

class SQLAlchemyResource(Resource):
    class ResourceObject:
        def __init__(self, resource, model, blacklist=None):
            self.resource = resource
            self.model = model
            self.blacklist = blacklist
            if self.blacklist is None:
                self.blacklist = []

        def id_(self):
            return str(
                self.model.__getattribute__(self.resource._primary_columns[0])
            )

        def type_(self):
            return self.resource.name()

        def attributes(self):
            attributes_ = alchemyjsonschema.dictify.jsonify(
                self.model, self.resource.schema
            )
            for key in self.blacklist:
                attributes_.pop(key, None)
            return attributes_

    def __init__(self, model_cls, sessionmaker):
        self._primary_columns = model_cls.__table__.primary_key.columns.keys()
        if len(self._primary_columns) > 1:
            raise NotImplementedError("Compound primary keys not supported")
        self.model_cls = model_cls
        self.model_primary_key = getattr(
            self.model_cls, self._primary_columns[0]
        )
        self.sessionmaker = sessionmaker
        self.session = sqlalchemy.orm.scoped_session(sessionmaker)
        self.blacklist = []
        factory = alchemyjsonschema.SchemaFactory(
            alchemyjsonschema.StructuralWalker
        )
        schema = factory(self.model_cls, excludes=self._primary_columns)
        super().__init__(schema)

    def _on_request_end(self):
        self.session.remove()

    def _id_filter(self, id_):
        return {self._primary_columns[0]: id_}

    def name(self):
        return inflection.camelize(
            self.model_cls.__name__, uppercase_first_letter=False
        )

    def exists(self, id_):
        return self.session.query(
            self.session.query(self.model_cls)
            .filter_by(**self._id_filter(id_))
            .exists()
        ).scalar()

    def create(self, attributes):
        model = self.model_cls(**attributes)
        self.session.add(model)
        self.session.commit()
        return SQLAlchemyResource.ResourceObject(
            self, model, blacklist=self.blacklist
        )

    def read(self, id_):
        model = (
            self.session.query(self.model_cls)
            .filter_by(**self._id_filter(id_))
            .one_or_none()
        )
        return (
            None
            if model is None
            else SQLAlchemyResource.ResourceObject(
                self, model, blacklist=self.blacklist
            )
        )

    def update(self, id_, attributes):
        model = (
            self.session.query(self.model_cls)
            .filter_by(**self._id_filter(id_))
            .one()
        )
        for k, v in attributes.items():
            setattr(model, k, v)
        self.session.merge(model)
        self.session.commit()
        return SQLAlchemyResource.ResourceObject(
            self, model, blacklist=self.blacklist
        )

    def delete(self, id_):
        # TODO shouldnt it be
        #  http://docs.sqlalchemy.org/en/latest/core/tutorial.html#deletes
        r = (
            self.session.query(self.model_cls)
            .filter_by(**self._id_filter(id_))
            .delete()
        )
        self.session.commit()
        return r

    def list_count(self):
        return self.session.query(
            sqlalchemy.func.count(self.model_primary_key)
        ).scalar()

    def list_(self, limit=0, page=0):
        if limit > 0:
            start = abs(page) * limit
            stop = start + limit
            models = self.session.query(self.model_cls).slice(start, stop)
        else:
            models = self.session.query(self.model_cls)
        res = []
        for model in models:
            res.append(
                SQLAlchemyResource.ResourceObject(
                    self, model, blacklist=self.blacklist
                )
            )
        return res

try:
    import dbapiext
except ImportError:
    pass

@gen.coroutine
def dbapi2Cursor(connection, transaction=False):
    @contextmanager
    def wrapper(connection, transaction):
        cursor = connection.cursor()
        try:
            yield cursor
            if transaction:
                connection.commit()
        except:
            if transaction:
                connection.rollback()
            raise
        finally:
            cursor.close()

    return wrapper(connection, transaction)

@gen.coroutine
def momokoCursor(pool, transaction=False):
    class wrapper:
        def __init__(self, pool, connection, transaction):
            self.pool = pool
            self.connection = connection
            self.transaction = transaction

        def __enter__(self):
            return self.connection

        @gen.coroutine
        def __exit__(self, type_, value, traceback):
            if self.transaction:
                if type_ is None:
                    yield self.connection.execute("COMMIT")
                else:
                    yield self.connection.execute("ROLLBACK")
            self.pool.putconn(self.connection)

    connection = yield pool.getconn(ping=False)
    if transaction:
        yield connection.execute("BEGIN")
    return wrapper(pool, connection, transaction)

class DBAPI2Resource(Resource):
    _types_mapping = {
        "boolean": "boolean",
        "integer": "integer",
        "number": "numeric",
        "string": "text",
    }

    class ResourceObject:
        def __init__(self, resource, row):
            self._resource = resource
            assert row is not None
            self.row = row

        def id_(self):
            return str(self.row[-1])

        def type_(self):
            return self._resource.name()

        def attributes(self):
            return {
                n: v for (n, v) in zip(self._resource.columns, self.row[:-1])
            }

    def __init__(self, schema, dbapi, connection):
        self.cursor = dbapi2Cursor
        self.connection = connection
        self.dbapi = dbapi
        if dbapi.__name__ == "momoko":
            self.cursor = momokoCursor
            self.dbapi = dbapi.psycopg2
        dbapiext.set_paramstyle(self.dbapi)
        self._tablename = inflection.pluralize(schema["title"])
        super().__init__(schema)
        self.columns = list(self._schema()._properties.keys())

    def _is_sqlite(self):
        return self.dbapi.__name__ == "sqlite3"

    def _is_postgresql(self):
        return self.dbapi.__name__ == "psycopg2"

    def _create_primary_key(self):
        id_type = "integer"
        if self._is_postgresql():
            id_type = "bigserial"
        key = "id {} not null primary key".format(id_type)
        if self._is_sqlite():
            key += " autoincrement"
        return key

    @gen.coroutine
    def _create_table(self):
        types = [
            self._types_mapping[self._schema.propinfo(c)["type"]] +
            (" not null" if c in self._schema.__required__ else "")
            for c in self.columns
        ]
        column_defs = [self._create_primary_key()] + [
            "{} {}".format(c, t) for c, t in zip(self.columns, types)
        ]
        with (yield self.cursor(self.connection, transaction=True)) as cursor:
            cur = dbapiext.execute_f(
                cursor,
                "create table if not exists %s (%s)",
                self._tablename,
                column_defs,
            )
            if is_future(cur):
                yield cur

    def name(self):
        return self.schema["title"]

    @gen.coroutine
    def exists(self, id_):
        with (yield self.cursor(self.connection)) as cursor:
            cur = dbapiext.execute_f(
                cursor, "select 1 from %s where id = %X", self._tablename, id_
            )
            if is_future(cur):
                cur = yield cur
            return cur.fetchone() is not None

    @gen.coroutine
    def create(self, attributes):
        with (yield self.cursor(self.connection, transaction=True)) as cursor:
            cur = dbapiext.execute_f(
                cursor,
                "insert into %s (%s) values (%X)",
                self._tablename,
                list(attributes.keys()),
                list(attributes.values()),
            )
            if is_future(cur):
                cur = yield cur
        with (yield self.cursor(self.connection)) as cursor:
            # XXX assuming id is monotonically increasing
            cur = dbapiext.execute_f(
                cursor,
                "select %s from %s order by id desc limit 1",
                self.columns + ["id"],
                self._tablename,
            )
            if is_future(cur):
                cur = yield cur
            return DBAPI2Resource.ResourceObject(self, cur.fetchone())

    @gen.coroutine
    def read(self, id_):
        with (yield self.cursor(self.connection)) as cursor:
            cur = dbapiext.execute_f(
                cursor,
                "select %s from %s where id = %X",
                self.columns + ["id"],
                self._tablename,
                id_,
            )
            if is_future(cur):
                cur = yield cur
            row = cur.fetchone()
            if not row:
                return None
            return DBAPI2Resource.ResourceObject(self, row)

    @gen.coroutine
    def update(self, id_, attributes):
        with (yield self.cursor(self.connection, transaction=True)) as cursor:
            cur = dbapiext.execute_f(
                cursor,
                "update %s set %X where id = %X",
                self._tablename,
                attributes,
                id_,
            )
            if is_future(cur):
                cur = yield cur
        with (yield self.cursor(self.connection)) as cursor:
            cur = dbapiext.execute_f(
                cursor,
                "select %s from %s where id = %X",
                self.columns + ["id"],
                self._tablename,
                id_,
            )
            if is_future(cur):
                cur = yield cur
            return DBAPI2Resource.ResourceObject(self, cur.fetchone())

    @gen.coroutine
    def delete(self, id_):
        with (yield self.cursor(self.connection, transaction=True)) as cursor:
            cur = dbapiext.execute_f(
                cursor, "delete from %s where id = %X", self._tablename, id_
            )
            if is_future(cur):
                cur = yield cur
            return cur.rowcount

    @gen.coroutine
    def list_(self, limit=0, page=0):
        with (yield self.cursor(self.connection)) as cursor:
            if limit > 0:
                start = abs(page) * limit
                cur = dbapiext.execute_f(
                    cursor,
                    "select %s from %s limit %d offset %d",
                    self.columns + ["id"],
                    self._tablename,
                    limit,
                    start,
                )
            else:
                cur = dbapiext.execute_f(
                    cursor,
                    "select %s from %s",
                    self.columns + ["id"],
                    self._tablename,
                )
            if is_future(cur):
                cur = yield cur
            rows = cur.fetchall()
            return [DBAPI2Resource.ResourceObject(self, row) for row in rows]

    @gen.coroutine
    def list_count(self):
        with (yield self.cursor(self.connection)) as cursor:
            cur = dbapiext.execute_f(
                cursor, "select count(1) from %s", self._tablename
            )
            if is_future(cur):
                cur = yield cur
            rows = cur.fetchone()
            return rows[0]


import math
import random
import numpy as np
from config_manager import config_manager
class CartPole:

    def __init__(self):
        config = config_manager()

        self.length = 0.5
        self.poleMass = 0.1
        self.cartMass = 1
        self.g = -9.8                    # Gravity

        self.poleSpeed = 0              # First temporal derivative of pole angle
        self.poleAcc = 0                # Second temporal derivative of pole angle
        self.cartPos = 0                # Position of the cart
        self.cartSpeed = 0              # Velocity of the cart
        self.cartAcc = 0                # Cart acceleration
        self.BB = 10                    # Bang bang force
        self.maxMag = 0.21              # Maximum angle before it fails
        self.minPos = -2.4
        self.maxPos = 2.4
        self.timeStep = 0.02
        self.episodeLength = 300

        self.poleAngle_bin = 0
        self.poleSpeed_bin = 0
        self.cartPos_bin = 0
        self.cartSpeed_bin = 0

        self.poleAngle = random.uniform(-self.maxMag+0.1, self.maxMag-0.1)  # Pole angle (theta)
        self.poleSpeed = 0

        self.actions = [-15, -10, -5, 0, 5, 10, 15]   # Predefined possible actions

        (self.length,
         self.poleMass,
         self.g,
         self.timeStep) = config.get_cartpole_params()

    def step(self, action):
        self.BB = action  # If the correct force value is passed (-F to F)
        # Algorithm from the actor-critic document:

        self.poleAcc = (self.g * np.sin(self.poleAngle) +
                            (np.cos(self.poleAngle)*(-self.BB-self.poleMass*self.length*self.poleSpeed**2*np.sin(self.poleAngle)))
                            /(self.poleMass + self.cartMass))\
                           /(self.length*(4/3 - (self.poleMass*np.cos(self.poleAngle)**2)/(self.poleMass + self.cartMass)))

        self.cartAcc = (self.BB + self.poleMass*self.length*(self.poleSpeed**2*np.sin(self.poleAngle) - self.poleAcc * np.cos(self.poleAngle)))\
                       /(self.poleMass + self.cartMass)

        self.poleSpeed = self.poleSpeed + self.timeStep*self.poleAcc

        self.cartSpeed = self.cartSpeed + self.timeStep*self.cartAcc

        self.poleAngle = self.poleAngle + self.timeStep*self.poleSpeed

        self.cartPos = self.cartPos + self.timeStep*self.cartSpeed

        reward = self.reward()

        return self.get_state(), self.get_moves(), reward

    # Returning the possible actions
    def get_moves(self):
        return self.actions

    def get_continuous_state(self):
        return [self.poleAngle, self.poleSpeed, self.cartPos, self.cartSpeed]

    def get_state(self):
        state = None  # The state object to be saved into
        # x: +-0.8, +- 2.4
        # theta: 0, +-1, +-2, +-12
        # cartSpeed: 0.5, inf
        # poleSpeed: 50, inf

        self.poleAngle_bin = np.digitize(self.poleAngle, [-0.24, -0.21, -0.18, -0.14, -0.10, -0.06, -0.03, 0, 0.03, 0.06, 0.10, 0.14, 0.18, 0.21, 0.24]) # 15 - 7 (best)
        self.poleSpeed_bin = np.digitize(self.poleSpeed, [-10, -8, -6, -4, -2, 0, 2, 4, 6, 8, 10])  # 11 - 6 (best)
        self.cartPos_bin = np.digitize(self.cartPos,     [-2.4, -1.5, -0.8, 0, 0.8, 1.5, 2.4])          # 7  4 (best)
        self.cartSpeed_bin = np.digitize(self.cartSpeed, [-10, -8, -6, -4, -2, 0, 2, 4, 6, 8, 10])  # 11 - 6 (best)

        state = [self.poleAngle_bin, self.poleSpeed_bin, self.cartPos_bin, self.cartSpeed_bin]
        return np.array(state)

    def reward(self):
        # reward = 0
        # if self.poleAngle_bin == 0:
        #     reward = 1
        # elif self.poleAngle_bin in [1, 2, 3, 4, 5]:
        #     reward = 0
        # else:
        #     reward = -10
        #if self.cartPos_bin > 4:
        #    cartPosDiff = 7 - self.cartPos_bin  # Offset
        #elif self.cartPos_bin <= 4:
        #    cartPosDiff = 4 - self.cartPos_bin  # Offset

        #if self.poleAngle_bin > 7:
        #    poleAngleDiff = 15 - self.poleAngle_bin   # Offset
        #elif self.poleAngle_bin <= 7:
        #    poleAngleDiff = 7 - self.poleAngle_bin    # Offset

        # reward = 12 - poleAngleDiff - cartPosDiff
        # return poleAngleDiff
        # return (1 - (self.cartPos ** 2) / 11.52 - (self.poleAngle ** 2) / 288)
        #

        ### THE GOLDEN BOI ###
        if abs(self.poleAngle) < 0.03:
           return 0 if self.poleSpeed < 0 else 1
        else:
           return 0 if self.poleAngle < 0 else 1

        # if abs(poleAngleDiff) < 1:
        #     return 0 if poleSpeedDiff > 0 else 1
        # else:
        #     return 0 if poleAngleDiff > 0 else 1
        #return reward

    def reset_problem(self):
        self.poleAngle = random.uniform(-self.maxMag, self.maxMag)  # Pole angle (theta)
        self.poleSpeed = 0              # First temporal derivative of pole angle
        self.poleAcc = 0                # Second temporal derivative of pole angle

        self.cartPos = 0                # Position of the cart
        self.cartSpeed = 0              # Velocity of the cart
        self.cartAcc = 0                # Cart acceleration

    def is_final(self, state=None):   # Continuous problem
        if np.abs(self.poleAngle) > self.maxMag:
            return True
        else:
            if np.abs(self.cartPos) >= self.maxPos:
                return True
            else:
                return False

    def get_continuous_poleAngle(self):
        return self.poleAngle


from BaseSpider.base_component.announcement_sub_resolver.ZGPR_Annoucement.ZGPR_CallBid.decompose.base.param_obtain import \
    ParamObtain, getmethod
from BaseSpider.base_component.announcement_sub_resolver.ZGPR_Annoucement.ZGPR_CallBid.decompose.source_datas import \
    SourceDatas

class ResourceFrom(ParamObtain):

    @classmethod
    @getmethod(priority=1)
    def get_from_xpath(cls):
        response = SourceDatas.response
        resource_form = response.xpath('//*[@class="p_o"]/*[contains(.,"信息来源")]//text()')
        if len(resource_form) > 1:
            return resource_form[1].replace(' ', '').replace('\xa0', '').replace('\t', '')
        else:
            return None

    @classmethod
    def check(cls, data: str):
        return True


import quantstats_modified as qs_mod
import os
qs_mod.extend_pandas()

def make_files(stock, etf, filepath, start_date=None, end_date=None, st=None):
    #stock = qs_mod.utils.download_returns(stock)
    #qs_mod.stats.sharpe(stock)
    #stock.sharpe()
    #qs_mod.reports.html(stock,
    #                etf,
    #                start_date,
    #                end_date,
    #                output=os.path.join(os.getcwd(),'quantstats-tearsheet3.html'))
    st.text(stock)
    st.text(etf)
    st.text(start_date)
    st.text(end_date)

    stock = qs_mod.utils.download_returns(stock)
    qs_mod.stats.sharpe(stock)
    stock.sharpe()

    # before this runs - it checks if this stock data is saved already
    # if not saved - run and make the data
    qs_mod.old_reports.html(stock, etf,
                            start_date=None,
                            end_date=None,
                            output=os.path.join(filepath,
                                        'quantstats-tearsheet3.html'),
                            filepath=filepath)
    # if it is saved - do nothing


import os
import tensorflow as tf
import numpy as np

from model import DCGAN

flags = tf.app.flags
flags.DEFINE_float("train_size", np.inf, "The size of train images [np.inf]")
flags.DEFINE_integer("batch_size", 64, "The size of batch images [64]")
flags.DEFINE_integer("input_height", 108, "The size of image to use (will be center cropped). [108]")
flags.DEFINE_integer("input_width", None,
                     "The size of image to use (will be center cropped). If None, same value as input_height [None]")
flags.DEFINE_integer("output_height", 64, "The size of the output images to produce [64]")
flags.DEFINE_integer("output_width", None,
                     "The size of the output images to produce. If None, same value as output_height [None]")
flags.DEFINE_string("dataset", "celebA", "The name of dataset [celebA, mnist, lsun]")
flags.DEFINE_string("input_fname_pattern", "*.jpg", "Glob pattern of filename of input images [*]")
flags.DEFINE_string("checkpoint_dir", "checkpoint", "Directory name to save the checkpoints [checkpoint]")
flags.DEFINE_string("sample_dir", "samples", "Directory name to save the image samples [samples]")
flags.DEFINE_boolean("crop", False, "True for training, False for testing [False]")
flags.DEFINE_integer("latent_dim", 100, "Number of images to generate during test. [100]")

# detect param
flags.DEFINE_float("lr", 0.01, "adam learning_rate")
flags.DEFINE_float("beta1", 0.9, "adam param beta1")
flags.DEFINE_float("beta2", 0.999, "adam param beta2")
flags.DEFINE_float("eps", 1e-8, "adam param eps")
flags.DEFINE_integer("Iter", 500, "iteration")

FLAGS = flags.FLAGS

def main(_):

    assert(os.path.exists(FLAGS.checkpoint_dir))

    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    with tf.Session(config=config) as sess:
        dcgan = DCGAN(
            sess,
            input_width=FLAGS.input_width,
            input_height=FLAGS.input_height,
            output_width=FLAGS.output_width,
            output_height=FLAGS.output_height,
            batch_size=FLAGS.batch_size,
            sample_num=FLAGS.batch_size,
            z_dim=FLAGS.latent_dim,
            dataset_name=FLAGS.dataset,
            input_fname_pattern=FLAGS.input_fname_pattern,
            crop=FLAGS.crop,
            checkpoint_dir=FLAGS.checkpoint_dir)

        dcgan.detect_anomaly(FLAGS)

if __name__ == '__main__':
    tf.app.run()


import os
import re
from setuptools import setup, find_packages

def get_version(source):
    """
    Retrieve the version of a python distribution.

    version_file default is the <project_root>/_version.py

    :param source: Path to project root
    :return: Version string
    """
    version_str_lines = open(os.path.join(source, '_version.py'), "rt").read()
    version_str_regex = r"^__version__ = ['\"]([^'\"]*)['\"]"
    mo = re.search(version_str_regex, version_str_lines, re.M)
    if mo:
        return mo.group(1)
    else:
        raise RuntimeError("Unable to find version string in %s." % os.path.join(source, os.path.basename(source)))

setup(
    name='Dockerizer',
    version=get_version('Dockerizer'),
    packages=find_packages(),
    install_requires=[
        'databasetools>=1.4.6',
        'PySimpleGUI>=3.39.0',
        'RuntimeWatch',
        'dirutility>=0.7.0'
    ],
    url='https://github.com/sfneal/Dockerizer',
    entry_points={
        'console_scripts': [
            'dockerizer = Dockerizer.dockerize:main',
            'morning-pull = Dockerizer.morning_pull:main',
            'docker-reset = Dockerizer.reset:main',
            'docker-clean = Dockerizer.clean:main',
            'docker-bootstrap = Dockerizer.bootstrap:main',
            'docker-reboot = Dockerizer.reboot:main'
        ]
    },
    license='',
    author='Stephen Neal',
    author_email='stephen@stephenneal.net',
    description='Docker CLI wrapper with GUI for quickly Dockerizing applications.'
)


# -*- coding: utf-8 -*-
import inspect
from datetime import timedelta
from typing import List, Union

import pandas as pd
from sqlalchemy import Column, Float, String, DateTime
from sqlalchemy.orm import Session

from zvt.api.data_type import Region, Provider
from zvt.contract import IntervalLevel
from zvt.utils.time_utils import date_and_time, is_same_time, now_pd_timestamp

class Mixin(object):
    id = Column(String, primary_key=True)
    # entity id for this model
    entity_id = Column(String)

    # the meaning could be different for different case,most of time it means 'happen time'
    timestamp = Column(DateTime)

    # __mapper_args__ = {
    #    "order_by": timestamp.desc()
    # }

    @classmethod
    def help(cls):
        print(inspect.getsource(cls))

    @classmethod
    def important_cols(cls):
        return []

    @classmethod
    def time_field(cls):
        return 'timestamp'

    @classmethod
    def register_recorder_cls(cls, region: Region, provider: Provider, recorder_cls):
        """
        register the recorder for the schema

        :param provider:
        :param recorder_cls:
        """
        # don't make provider_map_recorder as class field,it should be created for the sub class as need
        if not hasattr(cls, 'provider_map_recorder'):
            cls.provider_map_recorder = {}

        if region not in cls.provider_map_recorder:
            cls.provider_map_recorder[region] = {}
            cls.provider_map_recorder[region][provider] = recorder_cls

        elif provider not in cls.provider_map_recorder[region]:
            cls.provider_map_recorder[region][provider] = recorder_cls

    @classmethod
    def register_provider(cls, region: Region, provider: Provider):
        # dont't make providers as class field,it should be created for the sub class as need
        if not hasattr(cls, 'providers'):
            cls.providers = {}

        if region in cls.providers.keys():
            if provider not in cls.providers[region]:
                cls.providers[region].append(provider)
        else:
            cls.providers.update({region: [provider]})

    @classmethod
    def test_data_correctness(cls, region, provider, data_samples):
        for data in data_samples:
            item = cls.query_data(region=region, provider=provider, ids=[data['id']], return_type='dict')
            print(item)
            for k in data:
                if k == 'timestamp':
                    assert is_same_time(item[0][k], data[k])
                else:
                    assert item[0][k] == data[k]

    @classmethod
    def query_data(cls,
                   region: Region,
                   provider_index: int = 0,
                   ids: List[str] = None,
                   entity_ids: List[str] = None,
                   entity_id: str = None,
                   codes: List[str] = None,
                   code: str = None,
                   level: Union[IntervalLevel, str] = None,
                   provider: Provider = Provider.Default,
                   columns: List = None,
                   col_label: dict = None,
                   return_type: str = 'df',
                   start_timestamp: Union[pd.Timestamp, str] = None,
                   end_timestamp: Union[pd.Timestamp, str] = None,
                   filters: List = None,
                   session: Session = None,
                   order=None,
                   limit: int = None,
                   index: Union[str, list] = None,
                   time_field: str = 'timestamp',
                   func=None):
        from .api import get_data
        if provider == Provider.Default:
            provider = cls.providers[region][provider_index]
        return get_data(region=region, data_schema=cls, ids=ids, entity_ids=entity_ids, entity_id=entity_id, codes=codes,
                        code=code, level=level, provider=provider, columns=columns, col_label=col_label, return_type=return_type,
                        start_timestamp=start_timestamp, end_timestamp=end_timestamp, filters=filters, session=session,
                        order=order, limit=limit, index=index, time_field=time_field, fun=func)

    @classmethod
    def record_data(cls,
                    region: Region,
                    provider_index: int = 0,
                    provider: Provider = Provider.Default,
                    exchanges=None,
                    entity_ids=None,
                    codes=None,
                    batch_size=None,
                    force_update=None,
                    sleeping_time=None,
                    default_size=None,
                    real_time=None,
                    fix_duplicate_way=None,
                    start_timestamp=None,
                    end_timestamp=None,
                    close_hour=None,
                    close_minute=None,
                    one_day_trading_minutes=None,
                    **kwargs):
        if cls.provider_map_recorder:
            # print(f'{cls.__name__} registered recorders:{cls.provider_map_recorder}')
            if provider:
                recorder_class = cls.provider_map_recorder[region][provider]
            else:
                recorder_class = cls.provider_map_recorder[region][cls.providers[region][provider_index]]

            # get args for specific recorder class
            from zvt.contract.recorder import TimeSeriesDataRecorder
            if issubclass(recorder_class, TimeSeriesDataRecorder):
                args = [item for item in inspect.getfullargspec(cls.record_data).args if
                        item not in ('cls', 'region', 'provider_index', 'provider')]
            else:
                args = ['batch_size', 'force_update', 'sleeping_time']

            # just fill the None arg to kw,so we could use the recorder_class default args
            kw = {}
            for arg in args:
                tmp = eval(arg)
                if tmp is not None:
                    kw[arg] = tmp

            # FixedCycleDataRecorder
            from zvt.contract.recorder import FixedCycleDataRecorder
            if issubclass(recorder_class, FixedCycleDataRecorder):
                # contract:
                # 1)use FixedCycleDataRecorder to record the data with IntervalLevel
                # 2)the table of schema with IntervalLevel format is {entity}_{level}_[adjust_type]_{event}
                table: str = cls.__tablename__
                try:
                    items = table.split('_')
                    if len(items) == 4:
                        adjust_type = items[2]
                        kw['adjust_type'] = adjust_type
                    level = IntervalLevel(items[1])
                except:
                    # for other schema not with normal format,but need to calculate size for remaining days
                    level = IntervalLevel.LEVEL_1DAY

                kw['level'] = level

            # add other custom args
            for k in kwargs:
                kw[k] = kwargs[k]

            r = recorder_class(**kw)
            r.run()
        else:
            print(f'no recorders for {cls.__name__}')

class NormalMixin(Mixin):
    # the record created time in db
    created_timestamp = Column(DateTime, default=now_pd_timestamp(Region.CHN))
    # the record updated time in db, some recorder would check it for whether need to refresh
    updated_timestamp = Column(DateTime)

class Player(Mixin):
    # 参与者类型
    player_type = Column(String(length=64))
    # 所属国家
    country = Column(String(length=32))
    # 编码
    code = Column(String(length=64))
    # 名字
    name = Column(String(length=128))

class EntityMixin(Mixin):
    # 标的类型
    entity_type = Column(String(length=64))
    # 所属交易所
    exchange = Column(String(length=32))
    # 编码
    code = Column(String(length=64))
    # 名字
    name = Column(String(length=256))
    # 上市日
    list_date = Column(DateTime)
    # 退市日
    end_date = Column(DateTime)

    @classmethod
    def get_trading_dates(cls, start_date=None, end_date=None):
        """
        overwrite it to get the trading dates of the entity

        :param start_date:
        :param end_date:
        :return:
        """
        return pd.date_range(start_date, end_date, freq='B')

    @classmethod
    def get_trading_intervals(cls):
        """
        overwrite it to get the trading intervals of the entity

        :return:[(start,end)]
        """
        return [('09:30', '11:30'), ('13:00', '15:00')]

    @classmethod
    def get_interval_timestamps(cls, start_date, end_date, level: IntervalLevel):
        """
        generate the timestamps for the level

        :param start_date:
        :param end_date:
        :param level:
        """

        for current_date in cls.get_trading_dates(start_date=start_date, end_date=end_date):
            if level >= IntervalLevel.LEVEL_1DAY:
                yield current_date
            else:
                start_end_list = cls.get_trading_intervals()

                for start_end in start_end_list:
                    start = start_end[0]
                    end = start_end[1]

                    current_timestamp = date_and_time(the_date=current_date, the_time=start)
                    end_timestamp = date_and_time(the_date=current_date, the_time=end)

                    while current_timestamp <= end_timestamp:
                        yield current_timestamp
                        current_timestamp = current_timestamp + timedelta(minutes=level.to_minute())

    @classmethod
    def is_open_timestamp(cls, timestamp):
        timestamp = pd.Timestamp(timestamp)
        return is_same_time(timestamp, date_and_time(the_date=timestamp.date(),
                                                     the_time=cls.get_trading_intervals()[0][0]))

    @classmethod
    def is_close_timestamp(cls, timestamp):
        timestamp = pd.Timestamp(timestamp)
        return is_same_time(timestamp, date_and_time(the_date=timestamp.date(),
                                                     the_time=cls.get_trading_intervals()[-1][1]))

    @classmethod
    def is_finished_kdata_timestamp(cls, timestamp: pd.Timestamp, level: IntervalLevel):
        """
        :param timestamp: the timestamp could be recorded in kdata of the level
        :type timestamp: pd.Timestamp
        :param level:
        :type level: zvt.domain.common.IntervalLevel
        :return:
        :rtype: bool
        """
        timestamp = pd.Timestamp(timestamp)

        for t in cls.get_interval_timestamps(timestamp.date(), timestamp.date(), level=level):
            if is_same_time(t, timestamp):
                return True

        return False

    @classmethod
    def could_short(cls):
        """
        whether could be shorted

        :return:
        """
        return False

    @classmethod
    def get_trading_t(cls):
        """
        0 means t+0
        1 means t+1

        :return:
        """
        return 1

class NormalEntityMixin(EntityMixin):
    # the record created time in db
    created_timestamp = Column(DateTime, default=now_pd_timestamp(Region.CHN))
    # the record updated time in db, some recorder would check it for whether need to refresh
    updated_timestamp = Column(DateTime)

class Portfolio(EntityMixin):
    @classmethod
    def get_stocks(cls, region: Region, provider: Provider, timestamp, code=None, codes=None, ids=None):
        """
        the publishing policy of portfolio positions is different for different types,
        overwrite this function for get the holding stocks in specific date

        :param code: portfolio(etf/block/index...) code
        :param codes: portfolio(etf/block/index...) codes
        :param ids: portfolio(etf/block/index...) ids
        :param timestamp: the date of the holding stocks
        :param provider: the data provider
        :return:
        """
        from zvt.contract.api import get_schema_by_name
        schema_str = f'{cls.__name__}Stock'
        portfolio_stock = get_schema_by_name(schema_str)
        return portfolio_stock.query_data(region=region, provider=provider, code=code, codes=codes, timestamp=timestamp, ids=ids)

# 组合(Fund,Etf,Index,Block等)和个股(Stock)的关系 应该继承自该类
# 该基础类可以这样理解:
# entity为组合本身,其包含了stock这种entity,timestamp为持仓日期,从py的"你知道你在干啥"的哲学出发，不加任何约束
class PortfolioStock(Mixin):
    # portfolio标的类型
    entity_type = Column(String(length=64))
    # portfolio所属交易所
    exchange = Column(String(length=32))
    # portfolio编码
    code = Column(String(length=64))
    # portfolio名字
    name = Column(String(length=128))

    stock_id = Column(String)
    stock_code = Column(String(length=64))
    stock_name = Column(String(length=256))

# 支持时间变化,报告期标的调整
class PortfolioStockHistory(PortfolioStock):
    # 报告期,season1,half_year,season3,year
    report_period = Column(String(length=32))
    # 3-31,6-30,9-30,12-31
    report_date = Column(DateTime)

    # 占净值比例
    proportion = Column(Float)
    # 持有股票的数量
    shares = Column(Float)
    # 持有股票的市值
    market_cap = Column(Float)

__all__ = ['EntityMixin', 'Mixin', 'NormalMixin', 'NormalEntityMixin', 'Portfolio', 'PortfolioStock',
           'PortfolioStockHistory']


import logging

from rasa_sdk import constants, endpoint

logger = logging.getLogger(__name__)

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    logging.getLogger("matplotlib").setLevel(logging.WARN)

    # if you create custom actions, use a module path for the action package to
    # point the server to them e.g. "mymodule.actions"
    # noinspection PyTypeChecker
    endpoint.run(
        None,  # action package
        constants.DEFAULT_SERVER_PORT,  # port of the web server
        "*",  # cors origins
    )


import sys
import httplib
import xmltodict
from urlparse import urlparse

__copyright__ = "Copyright (c) 2016 EGI Foundation"
__license__ = "Apache Licence v2.0"

def get_vo_list():
    vos = []
    data = appdb_call('/rest/1.0/vos')
    for vo in data['appdb:appdb']['vo:vo']:
        vos.append(vo['@name'])
    return vos

def check_supported_VOs(data, vo):
    """
    Check if there are an image of the supported VO
    """
    if 'provider:image' in data['appdb:appdb']['virtualization:provider']:
        for os_tpl in data['appdb:appdb']['virtualization:provider']['provider:image']:
            if '@voname' in os_tpl and vo in os_tpl['@voname']:
                return True
    return False

def appdb_call(c):
    conn = httplib.HTTPSConnection('appdb.egi.eu')
    conn.request("GET", c)
    data = conn.getresponse().read()
    conn.close()
    data.replace('\n', '')
    return xmltodict.parse(data)

def get_sites(vo):
    data = appdb_call('/rest/1.0/sites?flt=%%2B%%3Dvo.name:%s&%%2B%%3Dsite.supports:1' % vo)
    providersID = []
    if 'appdb:site' in data['appdb:appdb']:
        if isinstance(data['appdb:appdb']['appdb:site'], list):
            sites = data['appdb:appdb']['appdb:site']
        else:
            sites = [data['appdb:appdb']['appdb:site']]
        for site in sites:
            if  isinstance(site['site:service'], list):
                for service in site['site:service']:
                    providersID.append(service['@id'])
            else:
                providersID.append(site['site:service']['@id'])

    # Get provider metadata
    endpoints = []
    for ID in providersID:
        data = appdb_call('/rest/1.0/va_providers/%s' % ID)
        if check_supported_VOs(data, vo):
            if ('provider:url' in data['appdb:appdb']['virtualization:provider'] and
                    data['appdb:appdb']['virtualization:provider']['@service_type'] == 'org.openstack.nova'):
                provider_name = data['appdb:appdb']['virtualization:provider']['provider:name']
                provider_endpoint_url = data['appdb:appdb']['virtualization:provider']['provider:url']
                url = urlparse(provider_endpoint_url)
                endpoints.append(provider_name + ";" + "%s://%s" % url[0:2])

    return endpoints

if __name__ == "__main__":
    vo = "fedcloud.egi.eu"
    option = "vos"
    if len(sys.argv) > 2:
        vo = sys.argv[2]
        option = sys.argv[1]
    elif len(sys.argv) > 1:
        option = sys.argv[1]

    if option == "vos":
        for vo in get_vo_list():
            print(vo)
    elif option == "sites":
        for site in get_sites(vo):
            if site.endswith("/"):
                print(site[:-1])
            else:
                print(site)
    else:
        print("Incorrect Option.")


import time
from functools import partial
from rptest.services.cluster import cluster
from rptest.clients.rpk import RpkTool, RpkException
from rptest.services.kafka_cli_consumer import KafkaCliConsumer
from ducktape.mark import parametrize
from rptest.services.admin import Admin
from rptest.util import expect_exception
from ducktape.utils.util import wait_until
from rptest.clients.types import TopicSpec
from rptest.tests.redpanda_test import RedpandaTest
from rptest.services.redpanda import RESTART_LOG_ALLOW_LIST
from rptest.services.redpanda_installer import RedpandaInstaller, wait_for_num_versions, ver_string

class OffsetRetentionDisabledAfterUpgrade(RedpandaTest):
    """
    When upgrading to Redpanda v23 or later offset retention should be disabled
    by default. Offset retention did not exist pre-v23, so existing clusters
    should have to opt-in after upgrade in order to avoid suprises.

    When a cluster upgrades to v23 then only newly committed offsets are
    reclaimed automatically. Offsets written prior to the upgrade must be
    manually removed using the kafka delete offset api.

    Offsets committed prior to v23 that are normally non-reclaimable become
    reclaimable if they are updated following the upgrade.
    """
    topics = (TopicSpec(), TopicSpec(), TopicSpec(), TopicSpec(), TopicSpec())

    # pre-defined named topics used to simulate use of topics that are used only
    # pre-v23, both pre and post upgrade, and only after upgrade to v23. the
    # idle variants stop receiving updates prior to enabling legacy support.
    pre_v23_topic = topics[0].name
    prepost_v23_topic = topics[1].name
    prepost_v23_topic_idle = topics[2].name
    post_v23_topic = topics[3].name
    post_v23_topic_idle = topics[4].name

    feature_config_timing = {
        "group_offset_retention_sec",
        "group_offset_retention_check_ms",
    }
    feature_config_legacy = "legacy_group_offset_retention_enabled"
    feature_config_names = feature_config_timing | {feature_config_legacy}

    def __init__(self, test_context):
        super(OffsetRetentionDisabledAfterUpgrade,
              self).__init__(test_context=test_context, num_brokers=3)
        self.installer = self.redpanda._installer

    def setUp(self):
        # handled by test case to support parameterization
        pass

    def _validate_pre_upgrade(self, version):
        """
        1. verify starting version of cluster nodes
        2. verify expected configs of initial cluster
        """
        self.installer.install(self.redpanda.nodes, version)
        super(OffsetRetentionDisabledAfterUpgrade, self).setUp()

        # wait until all nodes are running the same version
        unique_versions = wait_for_num_versions(self.redpanda, 1)
        assert ver_string(version) in unique_versions

        # sanity check none of feature configs should exist
        admin = Admin(self.redpanda)
        config = admin.get_cluster_config()
        assert config.keys().isdisjoint(self.feature_config_names)

    def _perform_upgrade(self, initial_version, version):
        """
        1. verify upgrade of all cluster nodes
        2. verify expected configs after upgrade
        """
        # upgrade all nodes to target version
        self.installer.install(self.redpanda.nodes, version)
        self.redpanda.restart_nodes(self.redpanda.nodes)

        # wait until all nodes are running the same version
        unique_versions = wait_for_num_versions(self.redpanda, 1)
        assert ver_string(initial_version) not in unique_versions

        # sanity check that all new feature configs exist
        admin = Admin(self.redpanda)
        config = admin.get_cluster_config()
        assert config.keys() > self.feature_config_names

        # configs should all have positive values
        for name in self.feature_config_timing:
            assert config[name] > 0

        # after upgrade legacy support should be disabled
        assert config[self.feature_config_legacy] == False

    def _offset_removal_occurred(self, period, pre_upgrade,
                                 pre_legacy_support):
        rpk = RpkTool(self.redpanda)
        group = "hey_group"

        def offsets_exist(include_idle):
            """
            Note to future ci-failure investigators. If you were to ever observe
            a failure in which it appeared that prepost/post topic offsets were
            not removed it could be because the following occurred:

              1) the version fence was not written for some reason
              2) so some of the offsets were incorrectly marked as legacy
              3) something else happened like updates to commits also failing

            it's unlikely that this would happen, but if it does, the proper fix
            would be to update the test to check for this condition (may be
            non-trivial to automate this check) and then retry the test.

            in real-world scenarios if this were to happen then it is expected
            that offset delete api is used to clean up any offsets that lose
            this race condition that exists during an upgrade.
            """
            desc = rpk.group_describe(group)
            if len(desc.partitions) == 0:
                return False

            pre = any(p.topic == self.pre_v23_topic for p in desc.partitions)
            prepost = any(p.topic == self.prepost_v23_topic
                          for p in desc.partitions)
            prepost_idle = any(p.topic == self.prepost_v23_topic_idle
                               for p in desc.partitions)
            post = any(p.topic == self.post_v23_topic for p in desc.partitions)
            post_idle = any(p.topic == self.post_v23_topic_idle
                            for p in desc.partitions)

            assert pre, "pre-topic offsets should never be reclaimed"

            # prior to upgrade, the only offsets that should exist are for the
            # pre and prepost topics.
            if pre_upgrade:
                assert not post, "post-topic has not yet been written"
                return prepost and prepost_idle

            if include_idle:
                return prepost and post and prepost_idle and post_idle
            return prepost and post

        # consume from the group for twice as long as the retention period
        # setting and verify that group offsets exist for the entire time.
        start = time.time()
        while time.time() - start < (period * 2):
            if pre_upgrade:
                # pre-v23
                rpk.produce(self.pre_v23_topic, "k", "v")
                rpk.consume(self.pre_v23_topic, n=1, group=group)
            else:
                # post-v23
                rpk.produce(self.post_v23_topic, "k", "v")
                rpk.consume(self.post_v23_topic, n=1, group=group)
                if pre_legacy_support:
                    rpk.produce(self.post_v23_topic_idle, "k", "v")
                    rpk.consume(self.post_v23_topic_idle, n=1, group=group)

            # pre/post-v23
            rpk.produce(self.prepost_v23_topic, "k", "v")
            rpk.consume(self.prepost_v23_topic, n=1, group=group)
            if pre_legacy_support:
                rpk.produce(self.prepost_v23_topic_idle, "k", "v")
                rpk.consume(self.prepost_v23_topic_idle, n=1, group=group)

            wait_until(lambda: offsets_exist(pre_legacy_support),
                       timeout_sec=1,
                       backoff_sec=1)
            time.sleep(1)

        # after one half life the offset should still exist. prior to legacy
        # support being enabled include idle / be strict. after it is enabled
        # then it's possible the idle topics get reclaimed so relax this check.
        time.sleep(period / 2)
        assert offsets_exist(pre_legacy_support)

        # after waiting for twice the retention period, it should be gone.
        # always be strict on this check and include idle topics.
        time.sleep(period * 2 - period / 2)
        return not offsets_exist(True)

    @cluster(num_nodes=3, log_allow_list=RESTART_LOG_ALLOW_LIST)
    @parametrize(initial_version=(22, 2, 9))
    @parametrize(initial_version=(22, 3, 11))
    def test_upgrade_from_pre_v23(self, initial_version):
        """
        1. test that retention feature doesn't work on legacy version
        2. upgrade cluster and test that feature still doesn't work
        3. enable legacy support on the cluster
        4. test that retention works properly
        """
        period = 30

        # in old cluster offset retention should not be active
        self._validate_pre_upgrade(initial_version)
        assert not self._offset_removal_occurred(period, True, True)

        # in cluster upgraded from pre-v23 retention should not be active
        self._perform_upgrade(initial_version, (23, 1))
        rpk = RpkTool(self.redpanda)
        rpk.cluster_config_set("group_offset_retention_sec", str(period))
        rpk.cluster_config_set("group_offset_retention_check_ms", str(1000))
        assert not self._offset_removal_occurred(period, False, True)

        # enable legacy. enablng legacy support takes affect at the next
        # retention check. since that is configured above to happen every second
        # then the response time should be adequate.
        rpk.cluster_config_set("legacy_group_offset_retention_enabled",
                               str(True))
        assert self._offset_removal_occurred(period, False, False)

class OffsetRetentionTest(RedpandaTest):
    topics = (TopicSpec(), )
    period = 30

    def __init__(self, test_context):
        # retention time is set to 30 seconds and expired offset queries happen
        # every second. these are likely unrealistic in practice, but allow us
        # to build tests that are quick and responsive.
        super(OffsetRetentionTest,
              self).__init__(test_context=test_context,
                             extra_rp_conf=dict(
                                 group_offset_retention_sec=self.period,
                                 group_offset_retention_check_ms=1000,
                             ))

        self.rpk = RpkTool(self.redpanda)

    @cluster(num_nodes=3)
    def test_offset_expiration(self):
        group = "hey_group"

        def offsets_exist():
            desc = self.rpk.group_describe(group)
            return len(desc.partitions) > 0

        # consume from the group for twice as long as the retention period
        # setting and verify that group offsets exist for the entire time.
        start = time.time()
        while time.time() - start < (self.period * 2):
            self.rpk.produce(self.topic, "k", "v")
            self.rpk.consume(self.topic, n=1, group=group)
            wait_until(offsets_exist, timeout_sec=1, backoff_sec=1)
            time.sleep(1)

        # after one half life the offset should still exist
        time.sleep(self.period / 2)
        assert offsets_exist()

        # after waiting for twice the retention period, it should be gone
        time.sleep(self.period * 2 - self.period / 2)
        assert not offsets_exist()

class OffsetDeletionTest(RedpandaTest):
    topics = (TopicSpec(partition_count=3), )
    group = "hey_hey_group"

    def __init__(self, test_context):
        super(OffsetDeletionTest, self).__init__(test_context=test_context)

        self.rpk = RpkTool(self.redpanda)

    def make_consumer(self, topic, session_timeout_ms=60000):
        return KafkaCliConsumer(
            self.test_context,
            self.redpanda,
            topic=topic,
            group=self.group,
            from_beginning=True,
            consumer_properties={'session.timeout.ms': session_timeout_ms},
            instance_name=f'cli-consumer-offset-delete-test-{topic}')

    @cluster(num_nodes=5)
    def test_offset_deletion(self):
        def wait_for_partitions_in_group(n):
            desc = self.rpk.group_describe(self.group)
            return len(desc.partitions) == n

        def assert_status(output, expected_status, expected_topic):
            for response in output:
                assert response.topic == expected_topic, f"Expected: {expected_topic} Observed: {response.topic}"
                assert response.status == expected_status, response.status

        # Produce some data to a topic and consume assigning new consumer group
        for _ in range(0, 10):
            self.rpk.produce(self.topic, "k", "v", partition=0)
            self.rpk.produce(self.topic, "k", "v", partition=1)
            self.rpk.produce(self.topic, "k", "v", partition=2)
        self.rpk.consume(self.topic, n=3, group=self.group)
        wait_until(partial(wait_for_partitions_in_group, 3),
                   timeout_sec=30,
                   backoff_sec=1)

        # Assert offset-delete errors when request contains missing topic-partitions
        missing_topic = f"{self.topic}-foo"
        missing_topic_partitions = {missing_topic: [0, 1, 2]}

        def assert_unknown_topic_or_partition(e):
            output = e.parsed_output
            assert len(output) == 3
            assert_status(output, 'UNKNOWN_TOPIC_OR_PARTITION', missing_topic)
            return True

        with expect_exception(RpkException, assert_unknown_topic_or_partition):
            self.rpk.offset_delete(self.group, missing_topic_partitions)

        # Assert offset-delete errors when non-existent group is passed in
        topic_partitions = {self.topic: [0, 1, 2]}
        with expect_exception(RpkException,
                              lambda e: 'GROUP_ID_NOT_FOUND' in str(e)):
            self.rpk.offset_delete("missing", topic_partitions)

        # Assert offset-delete errors when attempting to delete offsets of topic
        # partitions still assigned to an active group
        consumer = self.make_consumer(self.topic)
        consumer.start()
        consumer.wait_for_messages(1)
        wait_until(partial(wait_for_partitions_in_group, 3),
                   timeout_sec=30,
                   backoff_sec=1)

        def assert_group_already_subscribed(e):
            output = e.parsed_output
            assert len(output) == 3
            assert_status(output, 'GROUP_SUBSCRIBED_TO_TOPIC', self.topic)
            return True

        with expect_exception(RpkException, assert_group_already_subscribed):
            self.rpk.offset_delete(self.group, topic_partitions)

        consumer.stop()
        consumer.wait()
        consumer.free()

        # Assert offset-delete removes offsets of dead groups
        output = self.rpk.offset_delete(self.group, topic_partitions)
        assert len(output) == 3
        assert_status(output, 'OK', self.topic)

        desc = self.rpk.group_describe(self.group)
        assert len(desc.partitions) == 0

        # Assert offset-delete removes offsets of unsubscribed topic/partitions
        # within an active group
        new_topic = "foo"
        self.rpk.create_topic(new_topic, partitions=3)
        for i in range(0, 3):
            self.rpk.produce(new_topic, "k", "v", partition=i)

        # Add two new consumers to the group
        consumer_a = self.make_consumer(self.topic)
        min_default_group_session_timeout_ms = 6000
        consumer_b = self.make_consumer(new_topic,
                                        min_default_group_session_timeout_ms)
        consumer_a.start()
        consumer_b.start()

        # Wait until both have joined the group
        wait_until(partial(wait_for_partitions_in_group, 6),
                   timeout_sec=30,
                   backoff_sec=1)

        # Have them both consume some data
        consumer_a.wait_for_messages(1)
        consumer_b.wait_for_messages(1)

        # Shutdown one consumer
        consumer_b.stop()
        consumer_b.wait()

        # After consumer_b shuts down wait for the group rebalance to occur
        # and unsubscription of topic/partitions it was assigned to
        wait_until(partial(wait_for_partitions_in_group, 3),
                   timeout_sec=30,
                   backoff_sec=1)

        # Remove the offsets that it had been keeping in redpanda
        def validate_output(e):
            output = e.parsed_output
            assert len(output) == 4
            good_responses = [x for x in output if x.partition != 3]
            bad_responses = [x for x in output if x.partition == 3]
            assert len(good_responses) == 3
            assert len(bad_responses) == 1
            assert_status(good_responses, "OK", new_topic)
            assert_status(bad_responses, "UNKNOWN_TOPIC_OR_PARTITION",
                          new_topic)
            return True

        with expect_exception(RpkException, validate_output):
            self.rpk.offset_delete(self.group, {new_topic: [0, 1, 2, 3]})

        # Shutdown other consumer
        consumer_a.stop()
        consumer_a.wait()


import streamlit as st
import pandas as pd
import base64
import yfinance as yf

st.title('S&P 500 App')

st.markdown("""
This app retrieves the list of the **S&P 500** (from Wikipedia) and its corresponding **stock closing price** (year-to-date)!
* **Python libraries:** base64, pandas, streamlit, pandas
* **Data source:** [Wikipedia](https://en.jinzhao.wiki/wiki/List_of_S&P_500_companies).
""")

st.sidebar.header('User Input Features')

# Web scraping of S&P 500 data

df = pd.read_csv('SP500.csv')

sector = df.groupby('GICS Sector')

# Sidebar - Sector selection
sorted_sector_unique = sorted( df['GICS Sector'].unique() )
selected_sector = st.sidebar.multiselect('Sector', sorted_sector_unique, sorted_sector_unique)

# Filtering data
df_selected_sector = df[ (df['GICS Sector'].isin(selected_sector)) ]

st.header('Display Companies in Selected Sector')
st.write('Data Dimension: ' + str(df_selected_sector.shape[0]) + ' rows and ' + str(df_selected_sector.shape[1]) + ' columns.')
st.dataframe(df_selected_sector)

# Download S&P500 data
# https://discuss.streamlit.io/t/how-to-download-file-in-streamlit/1806
def filedownload(df):
    csv = df.to_csv(index=False)
    b64 = base64.b64encode(csv.encode()).decode()  # strings <-> bytes conversions
    href = f'Download CSV File'
    return href

st.markdown(filedownload(df_selected_sector), unsafe_allow_html=True)

# https://pypi.org/project/yfinance/

data = yf.download(
        tickers = list(df_selected_sector[:10].Symbol),
        period = "ytd",
        interval = "1d",
        group_by = 'ticker',
        auto_adjust = True,
        prepost = True,
        threads = True,
        proxy = None
    )

num_company = st.sidebar.slider('Number of Companies', 1, 5)

if st.button('Show Plots'):
    st.header('Stock Closing Price')
    for tickerSymbol in list(df_selected_sector.Symbol)[:num_company]:

	#get data on this ticker
        tickerData = yf.Ticker(tickerSymbol)
	# get the historical prices for this ticker

        tickerDf=tickerData.history(start='2011-6-22', end='2021-6-22')
        st.line_chart(tickerDf.Close)
        st.line_chart(tickerDf.Volume)


from __future__ import print_function, division
import sys
sys.path.append("../lib/ThinkBayes2/code/")
from thinkbayes2 import Suite, Pmf, SampleSum
import thinkplot
import simulationDD02

d6 = simulationDD02.Die(6)
three_exact = d6 + d6 + d6
best_attr_cdf = three_exact.Max(6)
best_attr_pmf = best_attr_cdf.MakePmf()

#plot
thinkplot.PrePlot(1)
thinkplot.Plot(best_attr_pmf)
thinkplot.Save(root='DD2',
               xlabel='Max Sum of 3 d6 over 6 attributes',
               ylabel='Probability',
               formats=['pdf'])

print("Program Finished")


#!/usr/local/bin/python3
#
# find_luddy.py : a simple maze solver
#
# Submitted by : [PUT YOUR NAME AND USERNAME HERE]
#
# Based on skeleton code by Z. Kachwala, 2019
#

import sys
import json

# Parse the map from a given filename
def parse_map(filename):
    with open(filename, "r") as f:
        return [[char for char in line] for line in f.read().split("\n")]

# Check if a row,col index pair is on the map
def valid_index(pos, n, m):
    return 0 <= pos[0] < n  and 0 <= pos[1] < m

# Find the possible moves from position (row, col)
def moves(map, row, col):
    moves=((row+1,col,'S'), (row-1,col,'N'), (row,col-1,'W'), (row,col+1,'E'))

    # Return only moves that are within the board and legal (i.e. on the sidewalk ".")
    return [ move for move in moves if valid_index(move, len(map), len(map[0])) and (map[move[0]][move[1]] in ".@" ) ]

# Perform search on the map
def search1(IUB_map):
    # Find my start position
    you_loc=[(row_i,col_i) for col_i in range(len(IUB_map[0])) for row_i in range(len(IUB_map)) if IUB_map[row_i][col_i]=="#"][0]
    fringe=[(you_loc,0)]
    visited=[]
    f3=[]
    f3.append(you_loc)

    while fringe:
        (curr_move, curr_dist)=fringe.pop(0)
        visited.append(curr_move)
        element=f3.pop(0)
        for move in moves(IUB_map, *curr_move):
            if IUB_map[move[0]][move[1]]=="@":
                f3.append((element,move))
                direction=''.join(filter(str.isalpha, str(f3.pop())))
                return [curr_dist + 1,direction]
            else:
                if (move[0:2] not in visited):
                    fringe.append((move[0:2], curr_dist + 1))
                    f3.append((element,move[2]))

# Main Function
if __name__ == "__main__":
    IUB_map=parse_map(sys.argv[1])
    print("Shhhh... quiet while I navigate!")
    solution = search1(IUB_map)
    print("Here's the solution I found:")
    if solution is None:
        print('Inf')
    else:
        print(solution[0],solution[1])


# -*- coding: utf-8 -*-
"""
Created on Wed Oct 21 16:51:49 2020

@author: Bryan Smith

file to test out created classes and functions
"""

from K9_TossnFetch import K9_TossnFetch as k9tf

k9 = k9tf()
k9.load()

dseason='13-18-FLL'
singleseason = k9.get_databyseason(dseason)

print(singleseason.head())
k=input("press close to exit")


import os, sys, glob, numpy
import matplotlib.pyplot as plot
from matplotlib import gridspec
from glbase3 import genelist, glload

TEs = glload('../../../../te_discovery/te_transcripts/transcript_table_merged.mapped.glb')

dfam = glload('../../../../te_discovery/dfam/dfam_annotation.glb') #, format={'force_tsv': True, 'name': 0, 'type': 3, 'subtype': 4})
dfam = {d['name']: '{0}:{1}:{2}'.format(d['type'], d['subtype'], d['name']) for d in dfam}

for filename in glob.glob('../../gls/*.glb'):
    print(filename)
    # Just do a simple annotation of yh the TE-contianig DE genes from the scRNA-seq
    stub = os.path.split(filename)[1].split('.')[0].replace('de_genes-', '')
    de_list = glload(filename)

    annot = de_list.map(genelist=TEs, key='transcript_id')

    for item in annot:
        item['TES'] = '; '.join(list(set([d['dom'] for d in item['doms']])))

    annot._optimiseData()

    annot = annot.getColumns(['ensg', 'transcript_id', 'name', 'gene_symbol', 'TES'])

    annot.save('{0}.glb'.format(stub))
    annot.saveTSV('{0}.tsv'.format(stub))