from future.backports.urllib.parse import urlparse

import logging

import cherrypy
import cherrypy_cors
from cherrypy import url
from jwkest import as_bytes
from jwkest import as_unicode
from oic.oauth2 import Message
from otest.events import EV_FAULT
from otest.events import EV_REQUEST
from otest.events import EV_RESPONSE
from otest.events import FailedOperation
from otest.events import Operation
from otest.flow import ABBR

from oidctest.cp import init_events
from oidctest.cp import write_events
from oidctest.cp.op_handler import init_keyjar
from oidctest.cp.op_handler import write_jwks_uri

logger = logging.getLogger(__name__)

def handle_error():
    cherrypy.response.status = 500
    cherrypy.response.body = [
        "Sorry, an error occured"
    ]

def set_content_type(resp, content_type):
    if ('Content-type', content_type) in resp.headers:
        return

    _headers = [h for h in resp.headers if h[0] != 'Content-type']
    _headers.append(('Content-type', content_type))
    resp.headers = _headers

def conv_response(op, resp):
    _stat = resp.status_code
    #  if self.mako_lookup and self.mako_template:
    #    argv["message"] = message
    #    mte = self.mako_lookup.get_template(self.mako_template)
    #    return [mte.render(**argv)]
    if _stat < 300:
        op.events.store(EV_RESPONSE, resp.message)
        cherrypy.response.status = _stat
        for key, val in resp.headers:
            cherrypy.response.headers[key] = val
        return as_bytes(resp.message)
    elif 300 <= _stat < 400:
        op.events.store('Redirect', resp.message)
        raise cherrypy.HTTPRedirect(resp.message, status=_stat)
    else:
        logger.debug("Error - Status:{}, message:{}".format(_stat, resp.message))
        op.events.store(EV_FAULT, resp.message)
        raise cherrypy.HTTPError(_stat, message=resp.message)

def store_request(op, where):
    _ev = Operation(where, path=url())

    try:
        op.events.store(EV_REQUEST, _ev)
    except Exception as err:
        raise

def parse_resource(resource):
    """

    :param resource:
    :return: A tuple: op_id and test_id
    """
    p = urlparse(resource)
    if p[0] == 'acct':
        loc, dom = p[2].split('@', 1)  # Should I check the domain part ?
        _x = loc.split('.')
        if len(_x) == 2:
            return _x
        elif len(_x) > 2:
            return '.'.join(_x[0:-1]), _x[-1]
        else:
            raise ValueError('Need both op_id and test_id, got {}'.format(_x))
    elif p[0] in ['http', 'https']:
        _x = p[2][1:].split('/')  # skip leading '/'
        if len(_x) >= 2:
            return _x[:2]  # only return the first two parts
        else:
            _txt = 'Need both op_id and test_id, got {}'.format(_x)
            logger.error(_txt)
            raise ValueError(_txt)
    else:
        return None

class WebFinger(object):
    def __init__(self, srv, version=''):
        self.srv = srv
        self.version = version

    @cherrypy.expose
    def index(self, resource, rel):
        ev = init_events('/.well-known/webfinger',
                         'Test tool version:{}'.format(self.version))
        ev.store(EV_REQUEST, Operation('WebFinger', resource=resource, rel=rel))

        if rel != 'http://openid.net/specs/connect/1.0/issuer':
            ev.store(EV_FAULT,
                     FailedOperation('Webfinger', error='unknown rel', rel=rel))
            try:
                op_id, test_id = parse_resource(resource)
            except (ValueError, TypeError):
                logger.error('webfinger resource specification faulty')
                raise cherrypy.HTTPError(
                    400, 'webfinger resource specification faulty')
            else:
                write_events(ev, op_id, test_id)
            raise cherrypy.NotFound()

        try:
            op_id, test_id = parse_resource(resource)
        except (ValueError, TypeError):
            logger.error('webfinger resource specification faulty')
            raise cherrypy.HTTPError(
                400, 'webfinger resource specification faulty')
        else:
            _path = '/'.join([op_id, test_id])

        cnf = cherrypy.request.config
        subj = resource
        _base = cnf['base_url']
        dummy = None

        # introducing an error
        if 'rp-discovery-webfinger-http-href' in resource:
            _base = _base.replace('https', 'http')
        if 'rp-discovery-webfinger-unknown-member' in resource:
            dummy = "foobar"

        if _base.endswith('/'):
            href = '{}{}'.format(_base, _path)
        else:
            href = '{}/{}'.format(_base, _path)

        ev.store(EV_RESPONSE,
                 Operation('Webfinger', href=href, subj=resource, dummy=dummy))
        write_events(ev, op_id, test_id)
        resp = self.srv.response(subj, href, dummy=dummy)
        cherrypy.response.headers['Content-Type'] = 'application/jrd+json'
        return as_bytes(resp)

class Configuration(object):
    @cherrypy.expose
    @cherrypy_cors.tools.expose_public()
    @cherrypy.tools.allow(
        methods=["GET", "OPTIONS"])
    def index(self, op):
        if cherrypy.request.method == "OPTIONS":
            logger.debug('Request headers: {}'.format(cherrypy.request.headers))
            cherrypy_cors.preflight(
                allowed_methods=["GET"],
                allowed_headers=['Authorization', 'content-type'],
                allow_credentials=True, origins='*'
            )
        else:
            store_request(op, 'ProviderInfo')
            resp = op.providerinfo_endpoint()
            if resp.status_code < 300:
                set_content_type(resp, 'application/json')
            return conv_response(op, resp)

class Registration(object):
    @cherrypy.expose
    @cherrypy_cors.tools.expose_public()
    @cherrypy.tools.allow(
        methods=["POST", "OPTIONS"])
    def index(self, op):
        if cherrypy.request.method == "OPTIONS":
            logger.debug('Request headers: {}'.format(cherrypy.request.headers))
            cherrypy_cors.preflight(
                allowed_methods=["POST"], origins='*',
                allowed_headers=['Authorization', 'content-type'])
        else:
            store_request(op, 'ClientRegistration')
            if cherrypy.request.process_request_body is True:
                _request = cherrypy.request.body.read()
            else:
                raise cherrypy.HTTPError(400,
                                         'Missing Client registration body')
            logger.debug('request_body: {}'.format(_request))
            resp = op.registration_endpoint(as_unicode(_request))
            if resp.status_code < 300:
                set_content_type(resp, 'application/json')
            return conv_response(op, resp)

class Authorization(object):
    @cherrypy.expose
    @cherrypy_cors.tools.expose_public()
    @cherrypy.tools.allow(
        methods=["GET", "OPTIONS"])
    def index(self, op, **kwargs):
        if cherrypy.request.method == "OPTIONS":
            cherrypy_cors.preflight(
                allowed_methods=["GET"], origins='*',
                allowed_headers=['Authorization', 'content-type'])
        else:
            store_request(op, 'AuthorizationRequest')
            logger.debug('AuthorizationRequest: {}'.format(kwargs))
            op.events.store(EV_REQUEST, kwargs)
            resp = op.authorization_endpoint(kwargs)
            return conv_response(op, resp)

class Token(object):
    _cp_config = {"request.methods_with_bodies": ("POST", "PUT")}

    @cherrypy.expose
    @cherrypy_cors.tools.expose_public()
    @cherrypy.tools.allow(
        methods=["POST", "OPTIONS"])
    def index(self, op, **kwargs):
        if cherrypy.request.method == "OPTIONS":
            cherrypy_cors.preflight(
                allowed_methods=["POST"], origins='*',
                allowed_headers=['Authorization', 'content-type'])
        else:
            store_request(op, 'AccessTokenRequest')
            try:
                authn = cherrypy.request.headers['Authorization']
            except KeyError:
                authn = None
            logger.debug('Authorization: {}'.format(authn))
            resp = op.token_endpoint(as_unicode(kwargs), authn, 'dict')
            if resp.status_code < 300:
                set_content_type(resp, 'application/json')
            return conv_response(op, resp)

class UserInfo(object):
    @cherrypy.expose
    @cherrypy_cors.tools.expose_public()
    @cherrypy.tools.allow(
        methods=["GET", "POST", "OPTIONS"])
    def index(self, op, **kwargs):
        if cherrypy.request.method == "OPTIONS":
            cherrypy_cors.preflight(
                allowed_methods=["GET", "POST"], origins='*',
                allowed_headers=['Authorization', 'content-type'])
        else:
            store_request(op, 'UserinfoRequest')
            args = {'request': kwargs}
            if cherrypy.request.process_request_body is True:
                _req = cherrypy.request.body.read()
                if _req:  # The request is either in kwargs or in body
                    args['request'] = _req

            try:
                args['authn'] = cherrypy.request.headers['Authorization']
            except KeyError:
                pass

            #kwargs.update(args)
            resp = op.userinfo_endpoint(**args)
            #if resp.status_code < 300:
            #    set_content_type(resp, 'application/json')
            return conv_response(op, resp)

class Claims(object):
    @cherrypy.expose
    def index(self, op, **kwargs):
        if cherrypy.request.method == "OPTIONS":
            cherrypy_cors.preflight(
                allowed_methods=["GET"], origins='*',
                allowed_headers='Authorization')
        else:
            try:
                authz = cherrypy.request.headers['Authorization']
            except KeyError:
                authz = None
            try:
                assert authz.startswith("Bearer")
            except AssertionError:
                op.events.store(EV_FAULT, "Bad authorization token")
                cherrypy.HTTPError(400, "Bad authorization token")

            tok = authz[7:]
            try:
                _claims = op.claim_access_token[tok]
            except KeyError:
                op.events.store(EV_FAULT, "Bad authorization token")
                cherrypy.HTTPError(400, "Bad authorization token")
            else:
                # one time token
                del op.claim_access_token[tok]
                _info = Message(**_claims)
                jwt_key = op.keyjar.get_signing_key()
                op.events.store(EV_RESPONSE, _info.to_dict())
                cherrypy.response.headers["content-type"] = 'application/jwt'
                return as_bytes(_info.to_jwt(key=jwt_key, algorithm="RS256"))

HTML_PRE = """
<!DOCTYPE html>

<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>OpenID Connect Relying Party Certification</title>
<link href="/static/bootstrap/css/bootstrap.min.css" rel="stylesheet">
<link href="/static/theme.css" rel="stylesheet">

OpenID Connect Relying Party Certification

"""

HTML_POST = """

"""

HTML_FOOTER = """

(C) 2017-2018 - OpenID
                            Foundation

E-mail: certification@oidf.org

Issues: Github

Version: {}

"""

def choice(profiles):
    keys = list(profiles.keys())
    keys.sort()

    line = [
        '
',
        ]
    for k in keys:
        line.append('
')
        line.append('  {}	'.format(k))
        line.append('  <input type="radio" name="profile" value="{}">	'.format(
            profiles[k]))
        line.append('
')
    line.append('
')
    return '\n'.join(line)

class Reset(object):
    def __init__(self, com_args, op_args):
        self.com_args = com_args
        self.op_args = op_args

    @cherrypy.expose
    def index(self, op):
        init_keyjar(op, self.op_args['keyjar'], self.com_args)
        write_jwks_uri(op, self.op_args, {})
        return b'OK'

class Root(object):
    def __init__(self, version=''):
        self.version = version

    @cherrypy.expose
    def index(self):
        response = [
            HTML_PRE,
            '
',
            '
',
            '    This is a tool for testing the compliance of an OpenID Connect Relying Party with the OpenID Connect specifications.',
            '    Before you start testing please read the ',
            '    Conformance Testing for RPs introduction guide.',
            '
',
            '
',

            '
',
            '
',
            '
Response Types
',
            '
',
            '
'
            '
For a list of OIDC RP library tests per response_type choose your preference:
',
            '    <form action="list" class="col-md-6">',
            choice(ABBR),
            '
',
            '            <button type="submit" class="btn btn-primary">Submit</button>',
            '
',
            '    </form>',
            '
',
            '
',

            HTML_FOOTER.format(self.version),

            HTML_POST
        ]
        return '\n'.join(response)

class Provider(Root):
    _cp_config = {'request.error_response': handle_error}

    def __init__(self, op_handler, flows, version=''):
        Root.__init__(self, version)
        self.op_handler = op_handler
        self.flows = flows
        self.configuration = Configuration()
        self.registration = Registration()
        self.authorization = Authorization()
        self.token = Token()
        self.userinfo = UserInfo()
        self.claims = Claims()
        self.reset = Reset(self.op_handler.com_args, self.op_handler.op_args)

    def _cp_dispatch(self, vpath):
        # Only get here if vpath != None
        ent = cherrypy.request.remote.ip
        logger.info('ent:{}, vpath: {}'.format(ent, vpath))

        if vpath[0] == 'static':
            return vpath

        if len(vpath) >= 2:
            ev = init_events(cherrypy.request.path_info,
                             'Test tool version:{}'.format(self.version))
            oper_id = vpath.pop(0)
            test_id = vpath.pop(0)

            # verify test_id
            try:
                self.flows[test_id]
            except KeyError:
                raise cherrypy.HTTPError(400, 'Unknown TestID')

            if len(vpath):
                if len(vpath) == 1:
                    endpoint = vpath.pop(0)
                    op = self.op_handler.get(oper_id, test_id, ev, endpoint)[0]
                    cherrypy.request.params['op'] = op
                    if endpoint == 'registration':
                        return self.registration
                    elif endpoint == 'authorization':
                        return self.authorization
                    elif endpoint == 'token':
                        return self.token
                    elif endpoint == 'userinfo':
                        return self.userinfo
                    elif endpoint == 'claim':
                        return self.claims
                    elif endpoint == 'reset':
                        return self.reset
                    else:  # Shouldn't be any other
                        raise cherrypy.NotFound()
                if len(vpath) == 2:
                    a = vpath.pop(0)
                    b = vpath.pop(0)
                    endpoint = '{}/{}'.format(a, b)
                    if endpoint == ".well-known/openid-configuration":
                        op = self.op_handler.get(oper_id, test_id, ev,
                                                 endpoint)[0]
                        cherrypy.request.params['op'] = op
                        return self.configuration

        return self

class RelyingPartyInstance(object):

    def rp_3rd_party_init_login(self, op, client_id):
        if client_id in op.cdb:
            if 'initiate_login_uri' in op.cdb[client_id]:
                raise cherrypy.HTTPRedirect(op.cdb[client_id]['initiate_login_uri'])
            else:
                response = ['
']
                response.append("no 3rd-party initiated login URI found for  " + client_id)
                response.append('
')
                return '\n'.join(response)
        else:
            raise cherrypy.HTTPError(404, "client_id " + client_id + " not found in client database!")

    @cherrypy.expose
    @cherrypy_cors.tools.expose_public()
    @cherrypy.tools.allow(
        methods=["GET", "OPTIONS"])
    def index(self, op, test_id, client_id):
        if cherrypy.request.method == "OPTIONS":
            logger.debug('Request headers: {}'.format(cherrypy.request.headers))
            cherrypy_cors.preflight(
                allowed_methods=["GET"],
                allowed_headers=['Authorization', 'content-type'],
                allow_credentials=True, origins='*'
            )
        else:
            if test_id == "rp-3rd_party-init-login":
                return self.rp_3rd_party_init_login(op, client_id)
            raise cherrypy.HTTPError(404, "no test handler found for test id: " + test_id)

class RelyingParty(object):
    def __init__(self, op_handler, version=''):
        self.version = version
        self.op_handler = op_handler
        self.instance = RelyingPartyInstance()

    @cherrypy.expose
    def index(self):
        response = [
            HTML_PRE,
            '
',
            '
',
            '    Add the Relying Party instance identifier to the path....',
            '
',
            '
',

            HTML_FOOTER.format(self.version),

            HTML_POST
        ]
        return '\n'.join(response)

    def _cp_dispatch(self, vpath):
        # Only get here if vpath != None
        ent = cherrypy.request.remote.ip
        logger.info('ent:{}, vpath: {}'.format(ent, vpath))

        if len(vpath) >= 1:
            ev = init_events(cherrypy.request.path_info,
                             'Test tool version:{}'.format(self.version))
            # e.g https://rp_test:8080/rp/mod_auth_openidc/rp-3rd_party-init-login/OLtakKrycmtj
            rp_id = vpath.pop(0)
            test_id = vpath.pop(0)
            client_id = vpath.pop(0)

            endpoint = ''

            op = self.op_handler.get(rp_id, test_id, ev,
                                     endpoint)[0]
            cherrypy.request.params['op'] = op
            cherrypy.request.params['test_id'] = test_id
            cherrypy.request.params['client_id'] = client_id

            return self.instance
        else:
            raise cherrypy.HTTPError(400, 'Unknown vpath stuffy')


from pyrogram import Client as bot
from pyrogram import filters
from extensions.database.database import database
import json

with open("extensions/reply/text.json","r",encoding="UTF-8") as data:
    text = json.load(data)

@bot.on_message(filters.private & filters.command("feedback"))
async def feedback(client,message):
    user_id = message.chat.id
    feedback = message.text.replace('/feedback','')
    for admin in database.get_admin():
        await client.send_message(admin,f"Feedback from {user_id}\n{feedback}")

@bot.on_message( filters.private & filters.command("users"))
async def users(client, message):
    users = database.users_num()
    await message.reply(text["status"].format(users))
@bot.on_message(filters.command("usernames"))
async def each_users(client,message):
    users = database.get_users()
    names = ""
    for usr in users:
        names+=f"{usr['name']} => @{usr['username']} => Joined in {usr['date']}\n"
await message.reply(names)


"""
Module to provide for an encapsulation of the text element.
"""

import logging
from typing import List, Optional, Tuple, cast

from pymarkdown.general.constants import Constants
from pymarkdown.general.parser_helper import ParserHelper
from pymarkdown.general.parser_logger import ParserLogger
from pymarkdown.general.position_marker import PositionMarker
from pymarkdown.tokens.indented_code_block_markdown_token import (
    IndentedCodeBlockMarkdownToken,
)
from pymarkdown.tokens.inline_markdown_token import InlineMarkdownToken
from pymarkdown.tokens.markdown_token import MarkdownToken
from pymarkdown.tokens.paragraph_markdown_token import ParagraphMarkdownToken
from pymarkdown.transform_gfm.transform_state import TransformState
from pymarkdown.transform_markdown.markdown_transform_context import (
    MarkdownTransformContext,
    RegisterHtmlTransformHandlersProtocol,
    RegisterMarkdownTransformHandlersProtocol,
)

POGGER = ParserLogger(logging.getLogger(__name__))

class TextMarkdownToken(InlineMarkdownToken):
    """
    Class to provide for an encapsulation of the text element.
    """

    # pylint: disable=too-many-arguments
    def __init__(
        self,
        token_text: str,
        extracted_whitespace: str,
        end_whitespace: Optional[str] = None,
        position_marker: Optional[PositionMarker] = None,
        line_number: int = 0,
        column_number: int = 0,
        is_special: bool = False,
        tabified_text: Optional[str] = None,
    ):
        (
            self.__token_text,
            self.__extracted_whitespace,
            self.__end_whitespace,
            self.__tabified_text,
        ) = (token_text, extracted_whitespace, end_whitespace, tabified_text)
        InlineMarkdownToken.__init__(
            self,
            MarkdownToken._token_text,
            "",
            position_marker=position_marker,
            line_number=line_number,
            column_number=column_number,
            is_special=is_special,
        )
        self.__compose_extra_data_field()

    # pylint: enable=too-many-arguments
    # pylint: disable=protected-access
    @staticmethod
    def get_markdown_token_type() -> str:
        """
        Get the type of markdown token for rehydration purposes.
        """
        return MarkdownToken._token_text

    # pylint: enable=protected-access

    def _set_token_text(self, new_text: str) -> None:
        self.__token_text = new_text
        self.__compose_extra_data_field()

    @property
    def token_text(self) -> str:
        """
        Returns the text associated with the token.
        """
        return self.__token_text

    @property
    def extracted_whitespace(self) -> str:
        """
        Returns any whitespace that was extracted before the processing of this element occurred.
        """
        return self.__extracted_whitespace

    @property
    def end_whitespace(self) -> Optional[str]:
        """
        Returns any whitespace that was extracted after the processing of this element occurred.
        """
        return self.__end_whitespace

    @property
    def tabified_text(self) -> Optional[str]:
        """
        Returns any text that had a tab character in it.
        """
        return self.__tabified_text

    def create_copy(self) -> "TextMarkdownToken":
        """
        Create a copy of this token.
        """
        return TextMarkdownToken(
            self.__token_text,
            self.__extracted_whitespace,
            self.__end_whitespace,
            line_number=self.line_number,
            column_number=self.column_number,
        )

    def __compose_extra_data_field(self) -> None:
        """
        Compose the object's self.extra_data field from the local object's variables.
        """

        data_field_parts = [self.__token_text, self.__extracted_whitespace]
        if self.__end_whitespace:
            data_field_parts.append(self.__end_whitespace)
            assert not self.__tabified_text
        elif self.__tabified_text:
            data_field_parts.extend(("", self.__tabified_text))
        self._set_extra_data(MarkdownToken.extra_data_separator.join(data_field_parts))

    def remove_final_whitespace(self) -> str:
        """
        Remove any final whitespace.  Used by paragraph blocks so that they do not
        end with a hard break.
        """

        removed_whitespace = ""
        # POGGER.debug("self.__tabified_text=:$:", self.__tabified_text)
        # POGGER.debug("self.__token_text=:$:", self.__token_text)
        (
            collected_whitespace_length,
            first_non_whitespace_index,
        ) = ParserHelper.collect_backwards_while_one_of_characters(
            self.__token_text, -1, Constants.ascii_whitespace
        )
        # POGGER.debug("collected_whitespace_length=:$:", collected_whitespace_length)
        # POGGER.debug("first_non_whitespace_index=:$:", first_non_whitespace_index)

        assert first_non_whitespace_index is not None
        if collected_whitespace_length:
            removed_whitespace = self.__token_text[
                first_non_whitespace_index : first_non_whitespace_index
                + collected_whitespace_length
            ]
            self.__token_text = self.__token_text[:first_non_whitespace_index]
            if self.__tabified_text:
                (
                    collected_whitespace_length,
                    first_non_whitespace_index,
                ) = ParserHelper.collect_backwards_while_one_of_characters(
                    self.__tabified_text, -1, Constants.ascii_whitespace
                )
                # POGGER.debug("collected_whitespace_length=:$:", collected_whitespace_length)
                # POGGER.debug("first_non_whitespace_index=:$:", first_non_whitespace_index)
                assert collected_whitespace_length is not None
                assert first_non_whitespace_index is not None
                removed_whitespace = self.__tabified_text[
                    first_non_whitespace_index : first_non_whitespace_index
                    + collected_whitespace_length
                ]
                self.__tabified_text = self.__tabified_text[:first_non_whitespace_index]
        self.__compose_extra_data_field()
        return removed_whitespace

    def combine(
        self, other_text_token: MarkdownToken, remove_leading_spaces: int
    ) -> str:
        """
        Combine the two text tokens together with a line feed between.
        If remove_leading_spaces > 0, then that many leading spaces will be
        removed from each line, if present.
        If remove_leading_spaces == -1, then.
        If remove_leading_spaces == 0, then.
        """

        if other_text_token.is_blank_line:
            text_to_combine = ""
            tabified_text_to_combine: Optional[str] = ""
            (
                whitespace_present,
                blank_line_sequence,
            ) = (
                other_text_token.extra_data,
                ParserHelper.replace_noop_character,
            )
        else:
            assert other_text_token.is_text
            text_other_token = cast(TextMarkdownToken, other_text_token)

            text_to_combine = text_other_token.token_text
            tabified_text_to_combine = text_other_token.tabified_text
            (
                whitespace_present,
                blank_line_sequence,
            ) = (
                text_other_token.extracted_whitespace,
                "",
            )

        removed_whitespace, prefix_whitespace = self.__combine_handle_whitespace(
            remove_leading_spaces, whitespace_present
        )

        if self.__tabified_text or tabified_text_to_combine:
            other_token_text = tabified_text_to_combine or text_to_combine

            this_token_text = self.__tabified_text or self.__token_text
            # POGGER.debug("this_token_text>:$:<", this_token_text)
            # POGGER.debug("blank_line_sequence>:$:<", blank_line_sequence)
            # POGGER.debug("prefix_whitespace>:$:<", prefix_whitespace)
            # POGGER.debug("other_token_text>:$:<", other_token_text)

            self.__tabified_text = (
                f"{this_token_text}{ParserHelper.newline_character}{blank_line_sequence}"
                + f"{prefix_whitespace}{other_token_text}"
            )
        self.__token_text = (
            f"{self.__token_text}{ParserHelper.newline_character}{blank_line_sequence}"
            + f"{prefix_whitespace}{text_to_combine}"
        )
        self.__compose_extra_data_field()
        return removed_whitespace

    def __combine_handle_whitespace(
        self, remove_leading_spaces: int, whitespace_present: Optional[str]
    ) -> Tuple[str, str]:
        prefix_whitespace = ""
        whitespace_to_append, removed_whitespace = None, ""
        if not remove_leading_spaces:
            assert whitespace_present is not None
            prefix_whitespace = whitespace_present
        elif remove_leading_spaces == -1:
            whitespace_to_append, prefix_whitespace = whitespace_present, ""
        else:
            assert whitespace_present is not None
            whitespace_present_size = len(whitespace_present)
            POGGER.debug(
                "whitespace_present>>$>>$<<",
                whitespace_present_size,
                whitespace_present,
            )
            POGGER.debug("remove_leading_spaces>>$<<", remove_leading_spaces)
            if whitespace_present_size < remove_leading_spaces:
                removed_whitespace, prefix_whitespace = whitespace_present, ""
            else:
                removed_whitespace, prefix_whitespace = (
                    whitespace_present[:remove_leading_spaces],
                    whitespace_present[remove_leading_spaces:],
                )

        if whitespace_to_append is not None:
            self.__extracted_whitespace = (
                f"{self.__extracted_whitespace}"
                + f"{ParserHelper.newline_character}{whitespace_to_append}"
            )
        return removed_whitespace, prefix_whitespace

    def register_for_markdown_transform(
        self, registration_function: RegisterMarkdownTransformHandlersProtocol
    ) -> None:
        """
        Register any rehydration handlers for leaf markdown tokens.
        """
        registration_function(
            TextMarkdownToken,
            TextMarkdownToken.__rehydrate_text,
            None,
        )

    @staticmethod
    def __rehydrate_text(
        context: MarkdownTransformContext,
        current_token: MarkdownToken,
        previous_token: Optional[MarkdownToken],
    ) -> str:
        """
        Rehydrate the text from the token.
        """
        _ = previous_token

        if (
            context.block_stack[-1].is_inline_link
            or context.block_stack[-1].is_inline_image
        ):
            return ""

        prefix_text = ""
        current_text_token = cast(TextMarkdownToken, current_token)
        POGGER.debug(
            f">>rehydrate_text>>:{ParserHelper.make_value_visible(current_text_token.token_text)}:<<"
        )
        # main_text = ParserHelper.resolve_noops_from_text(current_text_token.token_text)
        main_text = ParserHelper.remove_all_from_text(
            current_text_token.token_text, include_noops=True
        )

        POGGER.debug(f"<<rehydrate_text>>{ParserHelper.make_value_visible(main_text)}")

        POGGER.debug(
            f">>leading_whitespace>>:{ParserHelper.make_value_visible(current_text_token.extracted_whitespace)}:<<"
        )
        leading_whitespace = ParserHelper.remove_all_from_text(
            current_text_token.extracted_whitespace
        )
        POGGER.debug(
            f"<<leading_whitespace>>:{ParserHelper.make_value_visible(leading_whitespace)}:<<"
        )

        extra_line = ""
        assert context.block_stack
        if context.block_stack[-1].is_indented_code_block:
            code_block_token = cast(
                IndentedCodeBlockMarkdownToken, context.block_stack[-1]
            )
            (
                main_text,
                prefix_text,
                leading_whitespace,
            ) = TextMarkdownToken.__reconstitute_indented_text(
                main_text,
                code_block_token.extracted_whitespace,
                code_block_token.indented_whitespace,
                leading_whitespace,
            )
        elif context.block_stack[-1].is_html_block:
            extra_line = ParserHelper.newline_character
        elif context.block_stack[-1].is_paragraph:
            main_text = TextMarkdownToken.__reconstitute_paragraph_text(
                context, main_text, current_text_token
            )
        elif context.block_stack[-1].is_setext_heading:
            main_text = TextMarkdownToken.__reconstitute_setext_text(
                main_text, current_text_token
            )

        POGGER.debug(
            f"<<prefix_text>>{ParserHelper.make_value_visible(prefix_text)}"
            + f"<<leading_whitespace>>{ParserHelper.make_value_visible(leading_whitespace)}"
            + f"<<main_text>>{ParserHelper.make_value_visible(main_text)}<<"
        )
        return "".join([prefix_text, leading_whitespace, main_text, extra_line])

    @staticmethod
    def __reconstitute_paragraph_text(
        context: MarkdownTransformContext,
        main_text: str,
        current_token: "TextMarkdownToken",
    ) -> str:
        """
        For a paragraph block, figure out the text that got us here.
        """
        if ParserHelper.newline_character in main_text:
            owner_paragraph_token = cast(
                ParagraphMarkdownToken, context.block_stack[-1]
            )
            (
                main_text,
                owner_paragraph_token.rehydrate_index,
            ) = ParserHelper.recombine_string_with_whitespace(
                main_text,
                owner_paragraph_token.extracted_whitespace,
                owner_paragraph_token.rehydrate_index,
            )
            assert current_token.end_whitespace
            main_text, _ = ParserHelper.recombine_string_with_whitespace(
                main_text,
                current_token.end_whitespace,
                start_text_index=0,
                post_increment_index=True,
                add_whitespace_after=True,
            )
        return main_text

    @staticmethod
    def __reconstitute_indented_text(
        main_text: str,
        prefix_text: str,
        indented_whitespace: str,
        leading_whitespace: str,
    ) -> Tuple[str, str, str]:
        """
        For an indented code block, figure out the text that got us here.
        """
        recombined_text, _ = ParserHelper.recombine_string_with_whitespace(
            main_text,
            f"{prefix_text}{leading_whitespace}{indented_whitespace}",
            start_text_index=0,
            post_increment_index=True,
        )
        return f"{recombined_text}{ParserHelper.newline_character}", "", ""

    @staticmethod
    def __reconstitute_setext_text_item(
        text_part_index: int,
        text_part_value: str,
        rejoined_token_text: List[str],
        split_parent_whitespace_text: List[str],
    ) -> None:
        ws_prefix_text = ""
        ws_suffix_text = ""
        if split_parent_whitespace_text[text_part_index]:
            split_setext_text = split_parent_whitespace_text[text_part_index].split(
                ParserHelper.whitespace_split_character
            )
            split_setext_text_size = len(split_setext_text)
            if split_setext_text_size == 1:
                assert text_part_index == 0
                ws_suffix_text = split_setext_text[0]
                # if text_part_index == 0:
                #     ws_suffix_text = split_setext_text[0]
                # else:
                #     ws_prefix_text = split_setext_text[0]
            else:
                assert split_setext_text_size == 2
                ws_prefix_text = split_setext_text[0]
                ws_suffix_text = split_setext_text[1]

        rejoined_token_text.append(
            "".join([ws_prefix_text, text_part_value, ws_suffix_text])
        )

    @staticmethod
    def __reconstitute_setext_text(
        main_text: str, current_token: "TextMarkdownToken"
    ) -> str:
        """
        For a setext heading block, figure out the text that got us here.

        Because of the unique formatting of the setext data, the recombine_string_with_whitespace
        function cannot be used for this.
        """

        if ParserHelper.newline_character in main_text:
            split_token_text = main_text.split(ParserHelper.newline_character)
            assert current_token.end_whitespace is not None
            split_parent_whitespace_text = current_token.end_whitespace.split(
                ParserHelper.newline_character
            )

            rejoined_token_text: List[str] = []
            for text_part_index, text_part_value in enumerate(split_token_text):
                TextMarkdownToken.__reconstitute_setext_text_item(
                    text_part_index,
                    text_part_value,
                    rejoined_token_text,
                    split_parent_whitespace_text,
                )

            main_text = ParserHelper.newline_character.join(rejoined_token_text)
        else:
            POGGER.debug(f"main_text>>{ParserHelper.make_value_visible(main_text)}")
            POGGER.debug(
                f"current_token>>{ParserHelper.make_value_visible(current_token)}"
            )
            if current_token.end_whitespace and current_token.end_whitespace.endswith(
                ParserHelper.whitespace_split_character
            ):
                main_text = f"{current_token.end_whitespace[:-1]}{main_text}"
        return main_text

    @staticmethod
    def register_for_html_transform(
        register_handlers: RegisterHtmlTransformHandlersProtocol,
    ) -> None:
        """
        Register any functions required to generate HTML from the tokens.
        """
        register_handlers(
            TextMarkdownToken, TextMarkdownToken.__handle_text_token, None
        )

    @staticmethod
    def __handle_text_token(
        output_html: str,
        next_token: MarkdownToken,
        transform_state: TransformState,
    ) -> str:
        """
        Handle the text token.
        """
        text_token = cast(TextMarkdownToken, next_token)
        adjusted_text_token = ParserHelper.resolve_all_from_text(text_token.token_text)

        token_parts: List[str] = []
        if transform_state.is_in_code_block:
            POGGER.debug(
                "text_token.extracted_whitespace>:$:<", text_token.extracted_whitespace
            )
            leading_space = ParserHelper.resolve_all_from_text(
                text_token.extracted_whitespace
            )

            POGGER.debug("leading_space>:$:<", leading_space)
            POGGER.debug("adjusted_text_token>:$:<", adjusted_text_token)
            token_parts.extend(
                [
                    leading_space,
                    adjusted_text_token,
                ]
            )
        elif transform_state.is_in_html_block:
            POGGER.debug(
                "text_token.extracted_whitespace>:$:<", text_token.extracted_whitespace
            )
            leading_space = ParserHelper.resolve_all_from_text(
                text_token.extracted_whitespace
            )

            POGGER.debug("leading_space>:$:<", leading_space)
            POGGER.debug("adjusted_text_token>:$:<", adjusted_text_token)
            POGGER.debug("newline_character>:$:<", ParserHelper.newline_character)
            token_parts.extend(
                [
                    leading_space,
                    adjusted_text_token,
                    ParserHelper.newline_character,
                ]
            )
        elif transform_state.is_in_setext_block:
            token_parts.append(adjusted_text_token)
        else:
            TextMarkdownToken.__handle_text_token_normal(
                token_parts, text_token, adjusted_text_token
            )

        token_parts.insert(0, output_html)
        return "".join(token_parts)

    @staticmethod
    def __handle_text_token_normal(
        token_parts: List[str],
        text_token: "TextMarkdownToken",
        adjusted_text_token: str,
    ) -> None:
        POGGER.debug("adjusted_text_token>:$:<", adjusted_text_token)
        POGGER.debug("text_token.end_whitespace>:$:<", text_token.end_whitespace)
        if text_token.end_whitespace is not None:
            newlines_in_adjusted = ParserHelper.count_newlines_in_text(
                adjusted_text_token
            )

            resolved_whitespace = ParserHelper.resolve_all_from_text(
                text_token.end_whitespace
            )
            POGGER.debug("resolved_whitespace>:$:<", resolved_whitespace)
            newlines_in_whitespace = ParserHelper.count_newlines_in_text(
                resolved_whitespace
            )
            arrays_to_combine: List[List[str]] = []
            if newlines_in_adjusted == newlines_in_whitespace:
                arrays_to_combine.append(adjusted_text_token.split("\n"))
            else:
                TextMarkdownToken.__handle_text_token_normal_enhanced(
                    arrays_to_combine, text_token
                )

            arrays_to_combine.append(resolved_whitespace.split("\n"))
            assert len(arrays_to_combine[0]) == len(arrays_to_combine[1])
            POGGER.debug("arrays_to_combine>:$:<", arrays_to_combine)
            final_parts: List[str] = []
            for loop_index in range(len(arrays_to_combine[0]) * 2):
                loop_index_mod = loop_index % 2
                loop_index_div = loop_index // 2
                if loop_index_mod == 0 and final_parts:
                    final_parts.append("\n")
                POGGER.debug(
                    "$-->final_parts>:$:<...$,$",
                    loop_index,
                    final_parts,
                    loop_index_mod,
                    loop_index_div,
                )
                final_parts.append(arrays_to_combine[loop_index_mod][loop_index_div])
                POGGER.debug("final_parts>:$:<", final_parts)
            adjusted_text_token = "".join(final_parts)
        token_parts.append(adjusted_text_token)

    @staticmethod
    def __handle_text_token_normal_enhanced(
        arrays_to_combine: List[List[str]], text_token: "TextMarkdownToken"
    ) -> None:
        """
        This should only happen in the case where there are replacement characters that
        include replacing a character sequence with a newline.  Specifically target that.
        """

        assert "\a" in text_token.token_text
        replace_character_count = ParserHelper.count_characters_in_text(
            text_token.token_text, "\a"
        )
        assert replace_character_count % 3 == 0

        start_index = 0
        current_line = ""
        processed_lines: List[str] = []
        next_index, found_prefix = ParserHelper.collect_until_one_of_characters(
            text_token.token_text, start_index, "\a\n"
        )
        assert next_index is not None
        assert found_prefix is not None
        # POGGER.debug("next_index>:$:<", next_index)
        # POGGER.debug("found_prefix>:$:<", found_prefix)
        while next_index < len(text_token.token_text):
            current_line += found_prefix
            if text_token.token_text[next_index] == "\a":
                # POGGER.debug("before>:$:<", text_token.token_text[start_index:next_index])
                middle_index = text_token.token_text.find("\a", next_index + 1)
                # POGGER.debug("replace>:$:<", text_token.token_text[next_index+1:middle_index])
                start_index = text_token.token_text.find("\a", middle_index + 1)
                # POGGER.debug("with>:$:<", text_token.token_text[middle_index+1:start_index])
                current_line += text_token.token_text[middle_index + 1 : start_index]
            else:
                processed_lines.append(current_line)
                current_line = ""
                start_index = next_index
            next_index, found_prefix = ParserHelper.collect_until_one_of_characters(
                text_token.token_text, start_index + 1, "\a\n"
            )
            assert next_index is not None
            assert found_prefix is not None
            # POGGER.debug("next_index>:$:<", next_index)
            # POGGER.debug("found_prefix>:$:<", found_prefix)
        current_line += found_prefix

        # POGGER.debug("currernt_line>:$:<", current_line)
        assert len(current_line)
        processed_lines.append(current_line)

        arrays_to_combine.append(processed_lines)


import logging
import time
import os
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--dataname', type=str, default='flickr', help='Dataset name: flickr/coco/nuswide')
parser.add_argument('--bits', type=int, default=32, help='16/32/64/128')
parser.add_argument('--epochs', type=int, default=500, help='The epoch of training stage.')
# parser.add_argument('--gpuID', type=str, default='0', help='The GPU ID')
args = parser.parse_args()

if args.dataname == 'flickr':
    DIR = '../Data/raw_mir.mat'
elif args.dataname == 'nuswide':
    DIR = '../Data/raw_nus.mat'
elif args.dataname == 'coco':
    DIR = '../Data/raw_coco.mat'
else:
    print('Dataname Error!')
    DIR = ''

DATASET_NAME = args.dataname
CODE_LEN = args.bits
# GPU_ID = args.gpuID
FLAG_savecode = False

BETA = 0.6
LAMBDA1 = 0.1
LAMBDA2 = 0.1
NUM_EPOCH = 80
#NUM_EPOCH = 1

LR_IMG = 0.001
LR_TXT = 0.01
EVAL_INTERVAL = 40

BATCH_SIZE = 128 # 32
MU = 1.5
ETA = 0.4

MOMENTUM = 0.9
WEIGHT_DECAY = 5e-4

NUM_WORKERS = 1
EPOCH_INTERVAL = 2

MODEL_DIR = './checkpoint'
EVAL = False # EVAL = True: just test, EVAL = False: train and eval

logger = logging.getLogger('train')
logger.setLevel(logging.INFO)
now = time.strftime("%Y%m%d%H%M%S",time.localtime(time.time()))
log_name = now + '_log.txt'
log_dir = './log'
if not os.path.exists(log_dir):
    os.makedirs(log_dir)
txt_log = logging.FileHandler(os.path.join(log_dir, log_name))
txt_log.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
txt_log.setFormatter(formatter)
logger.addHandler(txt_log)

stream_log = logging.StreamHandler()
stream_log.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
stream_log.setFormatter(formatter)
logger.addHandler(stream_log)

logger.info('--------------------------Current Settings--------------------------')
logger.info('EVAL = %s' % EVAL)
logger.info('DATASET_NAME = %s' % DATASET_NAME)
logger.info('BETA = %.4f' % BETA)
logger.info('LAMBDA1 = %.4f' % LAMBDA1)
logger.info('LAMBDA2 = %.4f' % LAMBDA2)
logger.info('NUM_EPOCH = %d' % NUM_EPOCH)
logger.info('LR_IMG = %.4f' % LR_IMG)
logger.info('LR_TXT = %.4f' % LR_TXT)
logger.info('BATCH_SIZE = %d' % BATCH_SIZE)
logger.info('CODE_LEN = %d' % CODE_LEN)
logger.info('MU = %.4f' % MU)
logger.info('ETA = %.4f' % ETA)
logger.info('MOMENTUM = %.4f' % MOMENTUM)
logger.info('WEIGHT_DECAY = %.4f' % WEIGHT_DECAY)
# logger.info('GPU_ID =  %d' % GPU_ID)
logger.info('NUM_WORKERS = %d' % NUM_WORKERS)
logger.info('EPOCH_INTERVAL = %d' % EPOCH_INTERVAL)
logger.info('EVAL_INTERVAL = %d' % EVAL_INTERVAL)
logger.info('--------------------------------------------------------------------')


#-- imports --#
import requests as req
import json
import os
from logzero import logging

class Auth:

    def __init__(self):
        self.ROOT_API = "https://api.github.com"
        self.userName = ''
        self.token = ''
        self.SAVE_CREDENTIALS = False
        self.authenticated = False
        self.authenticate()

    def activeCredenitalsPresent(self):
        if(os.path.exists("data/secrets.json")):
            with open('data/secrets.json') as file:
                data = json.load(file)
                self.SAVE_CREDENTIALS = True
                if(self.checkAuth(data['userName'], data['token'])):
                    return True
                else:
                    return False

    def checkAuth(self, uName, token):
        r = req.get(self.ROOT_API + '/user', auth=(uName, token))
        if (r.status_code == 200):
            logging.info("Authentication success for:", r.json()['login'])
            self.userName = uName
            self.token = token
            return True
        else:
            return False

    def authenticate(self):
        if(self.activeCredenitalsPresent()):
            self.authenticated = True
            return {'userName': self.userName, 'token': self.token}

        while not self.authenticated:
            userName = str(input("Username: ")).strip()
            token = str(input("Token: ")).strip()
            r = req.get(self.ROOT_API + '/user', auth=(userName, token))
            if (r.status_code == 200):
                logging.info("Authentication success for:", r.json()['login'])
                self.authenticated = True
                credentials = {
                    "userName": userName,
                    "token": token
                }
                self.userName = userName
                self.token = token
                with open('data/secrets.json', 'w') as outfile:
                    json.dump(credentials, outfile)
                save = str(
                    input("Would you like your credentials saved for later (y/n): "))
                save = save.lower().strip()
                if (save == 'y' or save == 'yes'):
                    print('saving your credentials')
                    self.SAVE_CREDENTIALS = True
                else:
                    print("Credentials will be deleted after session")
                return {'userName': self.userName, 'token': self.token}
            else:
                logging.error('Failed to autheticate')
                statusCode = "Status: " + str(r.status_code)
                logging.error(statusCode)
                logging.error("Server Response:" + str(r.json()))


#-----------------------------------------------------------------------------------------------------
# Author: cgarcia@umw.edu
# Created: 9/12/2020
# About: This file provides uniform wrappers for each algorithm. Each algorithm wrapper takes a
#        BaseProblem and param dict as input and returns a solution as a dict in the form:
#        {'solution':<json-able dict containing solution attributes>, 'objective_f':<obj>, 'elapsed_time':time}
#
# Revision History:
# Date        Author            Change Description
# 9/12/2020   cgarcia           Initial version.
#-----------------------------------------------------------------------------------------------------

import alns
import ga
from structs import *
from functools import reduce

# Converts an ordered list of Nodes and a corresponding AssignmentSet object
# to a json-writable/readable solution in the following form:
# [{'worker':<worker id>, 'job_node':<job node id>, 'role':<role id>, 'start_time':<job start time>}]
def generic_solution_to_dict(job_nodes, assignment_set):
    aset = assignment_set
    assigned_job_nodes = [a for a in job_nodes if a.node_id in aset.scheduled_node_ids]
    a = reduce(lambda x,y: x+y, [[{'job_node':j.node_id, 'start_time':j.start_time, 'worker':a[0], 'role':a[1]}
                                  for a in aset.node_assignment_dict[j.node_id]] for j in assigned_job_nodes] + [[]])
    return {'assignments':a, 'reward':aset.reward}

# This method applies ALNS to the problem using the given parameters.
def alns_algorithm(prob, params):
    initial_sol = alns.ALNSEncodedSolution(prob)
    max_iter = params['max_iter']
    max_unimprove_time = params['max_unimprove_time']
    reaction_factor = params['reaction_factor']
    cost_f = params['cost_f']
    accept_method = params['accept_method']
    update_period = params['update_period']
    repair_fs = params['repair_fs']
    destroy_fs = params['destroy_fs']
    weight_deltas = params['weight_deltas']
    initial_sol = sorted([r(alns.ALNSEncodedSolution(prob)) for r in repair_fs], key=lambda x:cost_f(x))[0] # Start with best initial solution.
    sol = alns.alns(initial_sol, max_iter, max_unimprove_time, reaction_factor, cost_f,
                    accept_method, update_period, repair_fs, destroy_fs, weight_deltas)
    job_nodes = sol['solution'].job_nodes
    asmt = alns.decode(sol['solution'])
    return {'solution':generic_solution_to_dict(job_nodes, asmt),
            'objective_f':asmt.reward, 'elapsed_time':sol['elapsed_time']}


# ======================================================================
# No Space Left On Device
#   Advent of Code 2022 Day 07 -- Eric Wastl -- https://adventofcode.com
#
# Python implementation by Dr. Dean Earl Wright III
# ======================================================================

# ======================================================================
#                    t e s t _ s o l v e r . p y
# ======================================================================
"Test solver for Advent of Code 2022 day 07, No Space Left On Device"

# ----------------------------------------------------------------------
#                                                                 import
# ----------------------------------------------------------------------
import unittest

import aoc_07
import solver

# ----------------------------------------------------------------------
#                                                              constants
# ----------------------------------------------------------------------
EXAMPLE_TEXT = """
$ cd /
$ ls
dir a
14848514 b.txt
8504156 c.dat
dir d
$ cd a
$ ls
dir e
29116 f
2557 g
62596 h.lst
$ cd e
$ ls
584 i
$ cd ..
$ cd ..
$ cd d
$ ls
4060174 j
8033020 d.log
5626152 d.ext
7214296 k
"""
PART_ONE_TEXT = EXAMPLE_TEXT
PART_TWO_TEXT = EXAMPLE_TEXT

PART_ONE_RESULT = 95437
PART_TWO_RESULT = 24933642

# ======================================================================
#                                                             TestSolver
# ======================================================================

class TestSolver(unittest.TestCase):  # pylint: disable=R0904
    "Test Solver object"

    def test_empty_init(self):
        "Test the default Solver creation"

        # 1. Create default Solver object
        myobj = solver.Solver()

        # 2. Make sure it has the default values
        self.assertEqual(myobj.part2, False)
        self.assertEqual(myobj.text, None)
        self.assertEqual(myobj.device, None)

    def test_text_init(self):
        "Test the Solver object creation from text"

        # 1. Create Solver object from text
        myobj = solver.Solver(text=aoc_07.from_text(EXAMPLE_TEXT))

        # 2. Make sure it has the expected values
        self.assertEqual(myobj.part2, False)
        self.assertEqual(len(myobj.text), 23)
        self.assertNotEqual(myobj.device, None)
        self.assertEqual(len(myobj.device.root.dirs), 2)
        self.assertEqual(len(myobj.device.root.files), 2)

        # 3. Check methods
        self.assertEqual(myobj.device.root.size(), 48381165)
        self.assertEqual(myobj.device.root.sizes_under(100000), [94853, 584])
        self.assertEqual(myobj.device.root.sizes_over(600), [48381165, 94853, 24933642])
        self.assertEqual(myobj.device.root.sizes_over(600000), [48381165, 24933642])

    def test_part_one(self):
        "Test part one example of Solver object"

        # 1. Create Solver object from text
        myobj = solver.Solver(text=aoc_07.from_text(PART_ONE_TEXT))

        # 2. Check the part one result
        self.assertEqual(myobj.part_one(verbose=False), PART_ONE_RESULT)

    def test_part_two(self):
        "Test part two example of Solver object"

        # 1. Create Solver object from text
        myobj = solver.Solver(part2=True, text=aoc_07.from_text(PART_TWO_TEXT))

        # 2. Check the part two result
        self.assertEqual(myobj.part_two(verbose=False), PART_TWO_RESULT)

# ----------------------------------------------------------------------
#                                                  module initialization
# ----------------------------------------------------------------------
if __name__ == '__main__':
    pass

# ======================================================================
# end                   t e s t _ s o l v e r . p y                  end
# ======================================================================


import time #To implement delays
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities #to log network traffic
from playsound import playsound

Surrey_driver_licensing_label = "/html/body/div/div[1]/div/mat-dialog-container/app-search-modal/div[2]/div/div[3]/div[2]"
Surrey_claim_centre_label = "/html/body/div/div[1]/div/mat-dialog-container/app-search-modal/div[2]/div/div[4]/div"

Label = Surrey_claim_centre_label # Change this for different locations

# Chrome Profiles
options = webdriver.ChromeOptions()
options.add_argument('--user-data-dir=C:/Users/sshao/AppData/Local/Google/Chrome/User Data')
options.add_argument('--profile-directory=Profile 1')
#implement capabilities
capabilities = DesiredCapabilities.CHROME
# capabilities["loggingPrefs"] = {"performance": "ALL"}  # chromedriver < ~75
capabilities["goog:loggingPrefs"] = {"performance": "ALL"}  # chromedriver 75+

browser = webdriver.Chrome('C:\webdrivers\chromedriver',
options=options, desired_capabilities=capabilities) #Open Browser
browser.get("https://onlinebusiness.icbc.com/webdeas-ui/home") #Open ICBC Website

playsound('C:/Users/sshao/Desktop/icbc bot/ohyeah.mp3')

def scrape_date():
    # Click the Surrey label to refresh the data
    Surrey_label = browser.find_element_by_xpath(Label)
    Surrey_label.click()

    try:
        WebDriverWait(browser, 600).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, "div.date-title"))
        )
        print("waiting")
    finally:
        print("Date Located. Returning data: ")

    # Find earliest date
    earliest_date = browser.find_element_by_css_selector("div.date-title")
    date1 = earliest_date.text
    return(date1)
def press_next():
    nextbutton_location = "/html/body/app-root/app-home/mat-card/div[3]/div[2]/button"
    nextbutton = browser.find_element_by_xpath(nextbutton_location)
    nextbutton.click()
def getting_to_searching():
    # Fill in Last Name
    lastname_box_location = "/html/body/app-root/app-login/mat-card/mat-card-content/form/span[1]/div[1]/mat-form-field/div/div[1]/div[3]/input"
    Lastname = "Shao"
    lastnamebox = browser.find_element_by_xpath(lastname_box_location)
    lastnamebox.clear()
    lastnamebox.send_keys(Lastname)

    # Fill in Drivers License Number
    driverlicensenumber = "nah"
    licensebox_location = "/html/body/app-root/app-login/mat-card/mat-card-content/form/span[1]/div[2]/mat-form-field/div/div[1]/div[3]/input"
    licensebox = browser.find_element_by_xpath(licensebox_location)
    licensebox.clear()
    licensebox.send_keys(driverlicensenumber)

    # Fill in Keyword (Password)
    keyword = "nah"
    keywordbox_location = "/html/body/app-root/app-login/mat-card/mat-card-content/form/span[2]/div[1]/mat-form-field/div/div[1]/div[3]/input"
    keywordbox = browser.find_element_by_xpath(keywordbox_location)
    keywordbox.clear()
    keywordbox.send_keys(keyword)

    #check box
    checkbox = browser.find_element_by_xpath("/html/body/app-root/app-login/mat-card/mat-card-content/form/span[2]/div[3]/mat-checkbox/label/div")
    checkbox.click()

    #Sign in
    signinbox = browser.find_element_by_xpath("/html/body/app-root/app-login/mat-card/mat-card-content/form/div[3]/button[2]")
    signinbox.click()

    try:
        WebDriverWait(browser, 600).until(
        EC.presence_of_element_located((By.XPATH, "/html/body/app-root/app-driver/div/mat-card/div[5]/div[1]/app-appointments/div/div[2]/div/div[4]/button[1]"))
    )
    finally:
        print("Reschedule appointment button located. Delaying 0.1s, then pressing.")
    time.sleep(0.1)
    #reschedule appointmnet
    reschedule_button = browser.find_element_by_xpath("/html/body/app-root/app-driver/div/mat-card/div[5]/div[1]/app-appointments/div/div[2]/div/div[4]/button[1]")
    reschedule_button.click()

    try:
        WebDriverWait(browser, 600).until(
        EC.presence_of_element_located((By.XPATH, "/html/body/div/div[2]/div/mat-dialog-container/app-cancel/div/div/div[2]/button[2]"))
    )
    finally:
        print("Confirm button located. Delaying 0.1s, then pressing.")
    time.sleep(0.1)
    #confirm
    confirm_button = browser.find_element_by_xpath("/html/body/div/div[2]/div/mat-dialog-container/app-cancel/div/div/div[2]/button[2]")
    confirm_button.click()

    try:
        WebDriverWait(browser, 600).until(
        EC.presence_of_element_located((By.XPATH, "/html/body/div/div[1]/div/mat-dialog-container/app-search-modal/div/div/form/div[1]/div[1]/mat-form-field/div/div[1]/div[3]/input"))
    )
    finally:
        print("Page Loaded. Delaying 0.1s, then entering data.")
    time.sleep(0.1)

    #Fill in Location
    surrey_location_first_letter = "S"
    surrey_location_second_letter = "u"
    surrey_location_third_letter = "r"
    surrey_location_fifth_letter = "e"
    location_box = browser.find_element_by_xpath("/html/body/div/div[1]/div/mat-dialog-container/app-search-modal/div/div/form/div[1]/div[1]/mat-form-field/div/div[1]/div[3]/input")
    location_box.clear()
    location_box.send_keys(surrey_location_first_letter)
    time.sleep(0.1)
    location_box.send_keys(surrey_location_second_letter)
    time.sleep(0.1)
    location_box.send_keys(surrey_location_third_letter)
    time.sleep(0.1)
    location_box.send_keys(surrey_location_third_letter)
    time.sleep(0.1)
    location_box.send_keys(surrey_location_fifth_letter)
    time.sleep(1)

    #click dropdown
    try:
        WebDriverWait(browser, 600).until(
        EC.presence_of_element_located((By.XPATH, "/html/body/div/div[2]/div/div"))
    )
    finally:
        print("Dropdown located. Delaying 0.1s, then pressing.")
    time.sleep(0.1)
    dropdown_result = browser.find_element_by_xpath("/html/body/div/div[2]/div/div")
    dropdown_result.click()

    #Search
    time.sleep(0.5)
    search_button = browser.find_element_by_xpath("/html/body/div/div/div/mat-dialog-container/app-search-modal/div/div/form/div[2]/button")
    search_button.click()

    time.sleep(3)
    #Check Surrey
    try:
        WebDriverWait(browser, 600).until(
        EC.presence_of_element_located((By.XPATH,Label))
    )
    finally:
        print("Surrey location located. Delaying 0.1s, then parsing.")
    time.sleep(0.1)
    Surrey_label = browser.find_element_by_xpath(Label)
    Surrey_label.click()

press_next()
getting_to_searching()

time.sleep(0.2)
#Collecting Reference Date (most likely today's date)
try:
    WebDriverWait(browser, 60).until(
    EC.presence_of_element_located((By.CSS_SELECTOR, "div.date-title"))
    )
    print("waiting")
finally:
    print("Element located. Delaying 0.1s, then parsing.")
time.sleep(0.1)
reference_date = browser.find_element_by_css_selector("div.date-title").text
print("This is the reference date: ")
print(reference_date)

# Random status condition
is_same_thing = False

while not is_same_thing:
    new_date = scrape_date()
    if new_date == reference_date:
        print("No change.")
    else:
        print("NEW DATE FOUND!")
        print(new_date)
        is_same_thing = True
        playsound('C:/Users/sshao/Desktop/icbc bot/ohyeah.mp3')
    time.sleep(5) # Wait 5 seconds then check again

print("Go register")

# Find later dates, change the 1 at the end in span
#browser.find_element_by_xpath("/html/body/div/div[2]/div/mat-dialog-container/app-eligible-tests/div/div[2]/mat-button-toggle-group/div/span[1]/div")


#!/usr/bin/python

'''
Intel License
'''

import os
import subprocess
import shutil

'''
For generating API document, this will copy java files located in webrtc stack to android client sdk repo
'''

THIS = os.path.dirname(os.path.abspath(__file__))
DOXYGEN_PATH = os.path.join(THIS, 'doxygen')

def cleanEnv():
    os.chdir(os.path.join(THIS, '..'))
    cmd = ['./gradlew', 'clean']
    subprocess.call(cmd)

    if os.path.exists(os.path.join(THIS, 'html')):
        shutil.rmtree(os.path.join(THIS, 'html'))

def genDoc():
    os.chdir(DOXYGEN_PATH)
    cmd = ['doxygen', 'doxygen_android.conf']
    subprocess.call(cmd)

if __name__ == '__main__':
    cleanEnv()
    genDoc()


import sys
import unittest
import os

import coverage

from flask_script import Manager

from app import create_app, db

manager = Manager(create_app)

@manager.command
def createdb(drop_first=False):
    """Creates the database."""
    if drop_first:
        db.drop_all()
    db.create_all()

@manager.command
def test():
    """Runs unit tests."""
    os.environ['FLACK_CONFIG'] = 'testing'
    omit_items = [
        'manage.py',
        '*venv*/*',
        '*/serverapp351/*'
    ]

    # start coverage engine
    cov = coverage.Coverage(branch=True)
    cov.start()

    # run tests
    tests = unittest.TestLoader().discover('.')
    ok = unittest.TextTestRunner(verbosity=2).run(tests).wasSuccessful()

    # print coverage report
    cov.stop()
    cov.report(omit=omit_items)

    sys.exit(0 if ok else 1)

if __name__ == '__main__':
    manager.run()


import openai
import cv2
from matplotlib import pyplot as plt
import json
import key
import numpy as np
import urllib.request
import serial

# print the original image before pixelation
#  cv2_imshow(image)
# this function prints the input image
def printImage(img):
    # we can select different
    rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    plt.imshow(rgb)

# this image will print a grid of images
def printImageGrid(imglst):
    rbg_img = None
    for i, img in enumerate(imglst):
      plt.subplot(1, 2, i+1)
      # COLOR_RGB2BGR565
      # COLOR_BGR2RGB
      rgb_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
      plt.imshow(rgb_img)
    return rgb_img

# this function pixelates the image to a given height and width
def pixelate(img, w, h):
    height, width = img.shape[:2]

    # Resize input to "pixelated" size
    # INTER_LINEAR is a bilinear interpolation resampling method that uses the distanceweighted
    # average of the four nearest pixel values to estimate a new pixel value
    temp = cv2.resize(img, (w, h), interpolation=cv2.INTER_LINEAR)

    # Initialize output image
    # determines the “nearest” neighboring pixel and assumes its intensity value
    return cv2.resize(temp, (width, height), interpolation=cv2.INTER_NEAREST)

def genImages():
  # import the openAI API key
  openai.api_key = key.OPENAI_KEY

  # implement the DALL E image generation
  imagePrompt = input("enter a fire idea for a picture: ") # this is where the image prompt will go

  # create the image using DALL E and the image prompt
  rawImages = openai.Image.create(
      prompt = imagePrompt,
      n=2,
      size = "1024x1024"
  )

  # initialize the image lists
  img_values = []
  # this loops through generated Dall-E URLs and generates a list of pictures
  for url_dict in rawImages['data']:
    img_url = url_dict['url']
    with urllib.request.urlopen(img_url) as url_response:
      image_data = url_response.read()
    image = np.asarray(bytearray(image_data), dtype=np.uint8)
    # imglst.append(cv2.imdecode(image, cv2.IMREAD_COLOR))
    image = cv2.imdecode(image, cv2.IMREAD_COLOR)
    # here, we would likely send the picture data to the arduino

    # img_values.append(np.asarray(bytearray(pixelate(image, 32, 32))))
    img_values.append(np.asarray(pixelate(image, 32, 32)))

  test_img1 = img_values[0]
  test_img2 = img_values[1]

  resized_img1 = cv2.resize(test_img1, (32, 32))
  resized_img2 = cv2.resize(test_img2, (32,32))

  concat_images = np.concatenate((resized_img1, resized_img2), axis = 1)
  # If you need to save the image one more time:
  cv2.imwrite('concat_img.jpg', concat_images)
  print(concat_images.tolist())

  return concat_images

def main():
  # If you need to generate new images uncomment:
  # genImages()

  img = cv2.imread("concat_img.jpg")
  out_rgbs = img.tolist()
  ser = serial.Serial('/dev/tty.usbmodem2101', 9600)
  json_string = json.dumps(out_rgbs)
  ser.write(json_string.encode())

if __name__ == "__main__":
  main()


from abc import abstractmethod
from typing import Iterable, Optional

import numpy as np
from init_args_serializer import Serializable
from rcsenv import JointLimitException, RcsSimEnv

import pyrado
from pyrado.environments.sim_base import SimEnv
from pyrado.spaces.base import Space
from pyrado.spaces.box import BoxSpace
from pyrado.spaces.empty import EmptySpace
from pyrado.tasks.base import Task
from pyrado.utils.data_types import RenderMode

def to_pyrado_space(space) -> [BoxSpace, EmptySpace]:
    """
    Convert the box space implementation from RcsPySim to the one of Pyrado.

    :param space: a space from RcsPySim
    :return: a Pyrado `BoxSpace` or an Pyrado`EmptySpace` if `None` was given
    """
    if space is None:
        return EmptySpace
    return BoxSpace(space.min, space.max, labels=space.names)

class RcsSim(SimEnv, Serializable):
    """Base class for RcsPySim environments. Uses Serializable to facilitate proper serialization."""

    def __init__(
        self,
        envType: str,
        task_args: dict,
        dt: float = 0.01,
        max_steps: int = pyrado.inf,
        init_state: Optional[np.ndarray] = None,
        checkJointLimits: bool = True,
        joint_limit_penalty: float = -1e3,
        state_mask_labels: Optional[Iterable[str]] = None,
        **kwargs,
    ):
        """
        Constructor

        .. note::
            The joint type (i.e. position or torque control) is set in the associated xml files in `RcsPySim/config`.

        :param envType: environment type name as defined on the C++ side
        :param task_args: arguments for the task construction
        :param dt: integration step size in seconds
        :param max_steps: max number of simulation time steps
        :param domain_param: initial domain param values
        :param init_state: initial state sampler can be a callable or one fixed state
        :param checkJointLimits: flags if the joint limits should be ignored or not passed to the C++ constructor
        :param joint_limit_penalty: cost returned on termination due to joint limits. This is a different from the
                                    state bounds since `RcsPySim` return an error when the joint limits are violated.
        :param state_mask_labels: if `None`, the complete observation space (defined on the C++ side) will be used as
                                  state space. Pass an iterable of stings to select dimensions from the observation
                                  space as the state space. This is also affects the tasks.
        :param kwargs: keyword arguments which are available for `RcsSim` on the C++ side. These arguments will not
                       be stored in the environment object, thus are saved e.g. when pickled.
        """
        if not isinstance(envType, str):
            raise pyrado.TypeErr(given=envType, expected_type=str)
        if not isinstance(task_args, dict):
            raise pyrado.TypeErr(given=task_args, expected_type=dict)
        if dt <= 0:
            raise pyrado.ValueErr(given=dt, g_constraint="0.0 (float)")

        Serializable._init(self, locals())

        # Initialize basic variables
        super().__init__(dt, max_steps)
        self._check_joint_limits = checkJointLimits

        # Create Rcs-based implementation (RcsSimEnv comes from the pybind11 module)
        self._sim = RcsSimEnv(dt=dt, envType=envType, checkJointLimits=self._check_joint_limits, **kwargs)

        # Setup the initial domain parameters
        self._domain_param = self._unadapt_domain_param(self._sim.domainParam)

        if joint_limit_penalty > 0:
            raise pyrado.ValueErr(given=joint_limit_penalty, le_constraint="0")
        self._joint_limit_penalty = joint_limit_penalty

        # Initial init state space is taken from C++ which can be overwritten by setting a new init_space
        self._init_space = to_pyrado_space(self._sim.initStateSpace)

        # By default, the state space is a subset of the observation space. Set this to customize in subclass.
        if state_mask_labels is None:
            self.state_mask = None
        else:
            # Mask / redefine the state space, e.g. to match the init state space. This is affects the tasks.
            self.state_mask = self.obs_space.create_mask(state_mask_labels)

        # Dummy initialization, must be set by the derived classes
        self.init_state = None
        self.task_args = task_args
        self._task = self._create_task(self.task_args)

    @property
    def state_space(self) -> Space:
        """Derives the state space from the observation space using _state_from_obs or state_mask."""
        obs_space = self.obs_space
        # Check if _state_from_obs was overridden
        if self._state_from_obs.__func__ != RcsSim._state_from_obs:
            return BoxSpace(self._state_from_obs(obs_space.bound_lo), self._state_from_obs(obs_space.bound_up), None)
        # Check if there is a state mask
        if self.state_mask is not None:
            return obs_space.subspace(self.state_mask)
        # Identical to obs space
        return obs_space

    @property
    def obs_space(self) -> Space:
        return to_pyrado_space(self._sim.observationSpace)

    @property
    def act_space(self) -> Space:
        return to_pyrado_space(self._sim.actionSpace)

    @property
    def init_space(self) -> Space:
        return self._init_space

    @init_space.setter
    def init_space(self, space: Space):
        if not isinstance(space, Space):
            raise pyrado.TypeErr(given=space, expected_type=Space)
        self._init_space = space

    @property
    def task(self) -> Task:
        return self._task

    @abstractmethod
    def _create_task(self, task_args: dict) -> Task:
        # Needs to implemented by subclasses
        raise NotImplementedError

    @property
    def domain_param(self) -> dict:
        return self._unadapt_domain_param(self._sim.domainParam)

    @domain_param.setter
    def domain_param(self, domain_param: dict):
        if not isinstance(domain_param, dict):
            raise pyrado.TypeErr(given=domain_param, expected_type=dict)
        # Update the internal parameters. The New domain parameters will be applied on reset().
        self._domain_param.update(domain_param)

        # Update task
        self._task = self._create_task(self.task_args)

    @classmethod
    def get_nominal_domain_param(cls):
        """
        Get the nominal a.k.a. default domain parameters.

        .. note::
            It is highly recommended to have the same values as in the associated physics config file (p<NAME>.xml),
            since the nominal domain parameters are not set explicitly from Pyrado (only when randomizing).
        """
        raise NotImplementedError

    def _state_from_obs(self, obs: np.ndarray) -> np.ndarray:
        """
        Retrieve the system state from the observation. In most cases, the system state is a part of the observation.
        This function is to be used when the observations include additional information.
        The default implementation is based off `self.state_mask` which is set in sub-classes of `RcsSim`.

        :param obs: observation from the environment
        :return: state of the environment
        """
        obs_ = obs.copy()
        if self.state_mask is not None:
            obs_ = obs_[self.state_mask]
        return obs_

    def _adapt_domain_param(self, params: dict) -> dict:
        """
        Changes the domain parameters before passing them to the Rcs simulation.
        One use case is for example the rolling friction coefficient which is usually given unit-less but the Vortex
        physics engine expects it to be multiplied with the body's curvature radius.

        :param params: domain parameters to adapt
        :return: adapted parameters
        """
        return params

    def _unadapt_domain_param(self, params: dict) -> dict:
        """
        Changes the domain parameters coming from to the Rcs simulation.

        .. note::
            This function is called from the constructor.

        :param params: domain parameters to revert the previously done adaptation
        :return: unadapted parameters
        """
        return params

    def _get_state(self, state_dict: dict):
        state_dict["domain_param"] = self.domain_param
        state_dict["init_state"] = self.init_state

    def _set_state(self, state_dict: dict, copying: bool = False):
        self.domain_param = state_dict["domain_param"]
        self.init_state = state_dict["init_state"]

    def _disturbance_generator(self) -> (np.ndarray, None):
        """Provide an artificial disturbance. For example a force on a body in the physics simulation."""
        return None

    def reset(self, init_state: np.ndarray = None, domain_param: dict = None) -> np.ndarray:
        # Reset time
        self._curr_step = 0

        # Reset the state
        if init_state is None:
            # Sample from the init state space
            init_state = self._init_space.sample_uniform()
        else:
            if not init_state.shape == self._init_space.shape:
                raise pyrado.ShapeErr(given=init_state, expected_match=self._init_space)

        # Reset the task
        self._task.reset(env_spec=self.spec)

        # Use stored domain parameters if not overwritten
        if domain_param is None:
            domain_param = self._domain_param

        # Forward to C++ implementation
        obs = self._sim.reset(domainParam=self._adapt_domain_param(domain_param), initState=init_state)
        self.state = self._state_from_obs(obs)

        return obs

    def step(self, act: np.ndarray) -> tuple:
        # Current reward depending on the state (before step) and the (unlimited) action
        remaining_steps = self._max_steps - (self._curr_step + 1) if self._max_steps is not pyrado.inf else 0
        self._curr_rew = self._task.step_rew(self.state, act, remaining_steps)

        # Apply actuator limits
        act = self.limit_act(act)

        # Get the disturbance to be applied on the Rcs side
        disturbance = self._disturbance_generator()

        # Dynamics are calculated in the Rcs simulation
        try:
            obs = self._sim.step(act, disturbance)
        except JointLimitException:
            # Joint limits exceeded! Return (obs, rew, done, info) directly after this failure.
            return self._sim.lastObservation, self._joint_limit_penalty, True, dict(t=self._curr_step * self._dt)

        self.state = self._state_from_obs(obs)  # only for the Python side

        self._curr_step += 1

        # Check if the task or the environment is done
        done = self._task.is_done(self.state)
        if self._curr_step >= self._max_steps:
            done = True

        if done:
            # Add final reward if done
            self._curr_rew += self._task.final_rew(self.state, remaining_steps)

        return obs, self._curr_rew, done, dict()

    def render(self, mode: RenderMode = RenderMode(text=True), render_step: int = 1):
        if self._curr_step % render_step == 0:
            # Call base class
            super().render(mode)

            # Forward to Rcs GUI
            if mode.video:
                self._sim.render()

    def save_config_xml(self, fileName: str):
        """
        Save environment configuration as xml file for use on the C++ side.

        :param fileName: output file name
        """
        self._sim.saveConfigXML(fileName)

    def get_body_position(self, bodyName: str, refFrameName: str, refBodyName: str) -> np.ndarray:
        """
        Get the position of a body in the simulators config graph.
        This function uses code coped from `Rcs` to transform the position depending on a reference frame and/or body.

        :param bodyName: name of the body in the graph
        :param refFrameName: name of the reference frame, pass '' to use world coordinates
        :param refBodyName: name of the reference body, pass '' to use world coordinates
        :return: x,y,z positions in a reference frame coordinates relative to a reference bodies
        """
        return self._sim.getBodyPosition(bodyName, refFrameName, refBodyName)

    def get_body_extents(self, bodyName: str, shapeIdx: int = 0) -> np.ndarray:
        """
        Get the dimensions of a body in the simulators config graph.
        This function uses code coped from `Rcs` to transform the position depending on a reference frame and/or body.

        .. note::
            Depending on the kind of shape (e.g. box, sphere, torus, ect.) the extends mean different things.

        :param bodyName: name of the body in the graph
        :param shapeIdx: index of the shape in the `Body` node, defaults to the first shape of the body
        :return: x,y,z positions in a reference frame coordinates relative to a reference bodies
        """
        return self._sim.getBodyExtents(bodyName, shapeIdx)


import logging
import pymongo
from pymongo import MongoClient
from pymongo.collection import Collection
from functools import reduce
from datetime import datetime, timedelta
import os
import numpy as np

try:
    profile
except NameError:
    profile = lambda x: x

class Orderbooks:

    time_of_start_snapshot = None
    actual_end = None
    corrupt_order_book_count = 0

    def __init__(
            self,
            *,
            orderbooks_collection: Collection,
            envId: int,
            exchange: str,
            market: str,
            depth: float,
            start: datetime,
            end: datetime,
    ):

        self.start = start
        self.end = end
        self.depth = depth

        assert start < end

        self.buy_orderbook = []
        self.sell_orderbook = []

        start_snapshot = Orderbooks.get_start_snapshot(
            envId,
            exchange,
            market,
            start,
            orderbooks_collection=orderbooks_collection)

        self.buy_orderbook = start_snapshot["buy"]
        self.sell_orderbook = start_snapshot["sell"]

        # Ensure both buy and sell OBs are sorted
        self.buy_orderbook.sort(reverse=True)
        self.sell_orderbook.sort(reverse=False)

        self.time_of_start_snapshot = start_snapshot["ts"]

        last_orderbook = Orderbooks.get_last_orderbook(
            envId,
            exchange,
            market,
            start,
            end,
            orderbooks_collection=orderbooks_collection)

        self.actual_end = last_orderbook["ts"]

        if __debug__:
            logging.debug('start_snapshot: %r', start_snapshot)
            logging.debug('actual_end: %r', self.actual_end)

        # Setup an iterator to drive the simulation
        self.iter = iter(
            orderbooks_collection.find(
                filter={
                    "e": envId,
                    "x": exchange,
                    "m": market,
                    "ts": {
                        "$gte": self.time_of_start_snapshot,
                        "$lt": end
                    },
                    "s": {
                        "$exists": True
                    },
                    "V": "V", # valid flag
                }
            )
        )

    """
            ).sort([
                ("e", pymongo.ASCENDING),
                ("x", pymongo.ASCENDING),
                ("m", pymongo.ASCENDING),
                ("ts", pymongo.ASCENDING),
            ])
    """

    def __str__(self):
        return str(self.__class__) + '\n' + '\n'.join(
            '{} = {}'.format(
                item,
                self.__dict__[item]) for item in self.__dict__)

    def __iter__(self):
        return self

    @staticmethod
    def get_start_snapshot(
        envId: int,
        exchange: str,
        market: str,
        start: datetime,
        orderbooks_collection
    ):

        earlier_snapshots = list(orderbooks_collection.find(
            filter={
                "e": envId,
                "x": exchange,
                "m": market,
                # Look for a snapshot within the last 3 hours
                "ts": {
                    "$lte": start,
                    "$gte": start - timedelta(milliseconds=3*3600000)
                },
                "s": True,
            }))

        if len(earlier_snapshots) > 0:

            # Work with the most recent one
            start_snapshot = reduce(
                lambda x, y: x if x['ts'] > y['ts'] else y,
                earlier_snapshots,
            )

        else:

            # Nothing earlier; look for the next snapshot
            snapshots = list(
                orderbooks_collection.find(
                    filter={
                        "e": envId,
                        "x": exchange,
                        "m": market,
                        "ts": {
                            "$gte": start,
                        },
                        "s": True,
                    }).sort("ts", 1).limit(1)
            )
            if len(snapshots) == 0:
                raise StopIteration
            else:
                start_snapshot = snapshots[0]

        return start_snapshot

    @staticmethod
    def get_first_orderbook(
        envId: int,
        exchange: str,
        market: str,
        start: datetime,
        orderbooks_collection,
    ):

        # Nothing earlier; look for the next snapshot
        orderbooks = list(
            orderbooks_collection.find(
                filter={
                    "e": envId,
                    "x": exchange,
                    "m": market,
                    "ts": {
                        "$gte": start,
                    },
                })
            .sort("ts", 1).limit(1)
        )
        if len(orderbooks) == 0:
            return None
        else:
            return orderbooks[0]

    @staticmethod
    def get_last_orderbook(
        envId: int,
        exchange: str,
        market: str,
        start: datetime,
        end: datetime,
        orderbooks_collection,
    ):

        # Nothing earlier; look for the next snapshot
        orderbooks = list(
            orderbooks_collection.find(
                filter={
                    "e": envId,
                    "x": exchange,
                    "m": market,
                    "ts": {
                        "$gt": start,
                        "$lt": end,
                    },
                }).sort("ts", -1).limit(1)
        )

        if len(orderbooks) == 0:
            return None
        else:
            return orderbooks[0]

    @staticmethod
    def count_orderbooks(
        envId: int,
        exchange: str,
        market: str,
        start: datetime,
        end: datetime,
        orderbooks_collection,
    ):

        return orderbooks_collection.count_documents(
            filter={
                "e": envId,
                "x": exchange,
                "m": market,
                "ts": {
                    "$gte": start,
                    "$lt": end,
                },
                "V": "V",
                "s": {
                    "$exists": True
                }
            })

    @staticmethod
    def count_corrupt_orderbooks(
        envId: int,
        exchange: str,
        market: str,
        start: datetime,
        end: datetime,
        orderbooks_collection,
    ):

        return orderbooks_collection.count_documents(
            filter={
                "e": envId,
                "x": exchange,
                "m": market,
                "ts": {
                    "$gte": start,
                    "$lte": end,
                },
                "V": "C",
            })

    @profile
    def next(self):

        while True: # Until a clean OB

            while True: # Until prelimary OBs are removed

                orderbook = self.iter.next()

                if orderbook == None:
                    raise StopIteration

                assert "s" in orderbook

                if orderbook["s"] == True:

                    # Snapshot
                    self.buy_orderbook = orderbook["buy"]
                    self.sell_orderbook = orderbook["sell"]

                else:

                    # Delta
                    self.apply_deltas(orderbook)

                    orderbook["buy"] = self.buy_orderbook
                    orderbook["sell"] = self.sell_orderbook

                # First orderbooks not needed
                if self.start.replace(tzinfo=None) <= orderbook["ts"].replace(tzinfo=None):
                    break

            # Sanity check: best buy strictly less than best sell
            try:
                assert orderbook["buy"][0][0] < orderbook["sell"][0][0]
                break

            except AssertionError as err:
                # Log the first instance, otherwise just keep a count
                if self.corrupt_order_book_count == 0:
                    msg = f'OB Corruption: bestBuy={orderbook["buy"][0][0]} >= bestSell={orderbook["sell"][0][0]} @ {orderbook["ts"]}'
                    logging.error (msg)
                self.corrupt_order_book_count += 1

        orderbook["buy"] = np.array(orderbook["buy"], dtype=float)
        orderbook["sell"] = np.array(orderbook["sell"], dtype=float)

        assert len(orderbook["buy"]) > 0
        assert len(orderbook["sell"]) > 0

        return orderbook

    def apply_deltas(self, orderbook):

        assert self.buy_orderbook != None
        assert self.sell_orderbook != None

        # Apply the deltas to both sides
        self.apply_delta(self.buy_orderbook, orderbook["buy"])
        self.apply_delta(self.sell_orderbook, orderbook["sell"])

        self.buy_orderbook.sort(reverse=True)
        self.sell_orderbook.sort(reverse=False)

        # Ensure buy_orderbook has unique entries
        assert len(self.buy_orderbook) == len(
            set([x[0] for x in self.buy_orderbook]))

        # Ensure sell_orderbook has unique entries
        assert len(self.sell_orderbook) == len(
            set([x[0] for x in self.sell_orderbook]))

    def apply_delta(self, orderbook, deltas):

        for delta in deltas:

            for idx, entry in enumerate(orderbook):
                if entry[0] == delta[1]:
                    del orderbook[idx]

            if delta[0] == 0:   # Insert
                orderbook.append([delta[1], delta[2]])

            elif delta[0] == 1:  # Remove
                pass

            elif delta[0] == 2:  # Update
                orderbook.append([delta[1], delta[2]])

    @staticmethod
    def apply_depth(depth: float, orderbook: np.array) -> np.array:

        if depth == 0:
            return orderbook

        i = 0

        for entry in np.cumsum(np.prod(orderbook, axis=1)):

            i += 1

            if entry >= depth:
                break

        return orderbook[0:i]


"""
Action for calculating first two principal component of the RFM table and add the value of those component to a new dataframe
Inputs:
    - segment_data: Dataframe with RFM values and segments
    - customer_id_column: Customer identification column of the segment_data
    - segment_column: Segment labels columns of segment_data
Output:
    - Modified Dataframe with RFM values and  the first two principal components values
"""""
import traceback
import pandas as pd
from dft import df_plot
from sklearn.preprocessing import StandardScaler
from dft.base_execution_handler import BaseExecutionHandler
from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder
import dirty_cat
import numpy as np

class ExecutionHandler(BaseExecutionHandler):
    def execute(self, segment_data, customer_id_column, segment_column):

        from sklearn.decomposition import PCA
        try:
            pca = PCA(2)  # calculating the first two principal component
            data = segment_data.drop([customer_id_column, segment_column], axis=1)
            data = self.numeric_column_scale(data)
            data = self.categorical_column_encode(data)
            df = pca.fit_transform(data)
            PC1 = data.columns[pca.components_[0].argmax()] + " (Feature 1)"
            PC2 = data.columns[pca.components_[1].argmax()] + " (Feature 2)"
            d = pd.DataFrame(df)
            d[segment_column] = segment_data[segment_column]
            d.rename(columns={0: PC1, 1: PC2}, inplace=True)
            table = pd.pivot(d, values=PC2,
                             columns=segment_column).reset_index()
            n = len(d[segment_column].unique())
            table[PC1] = d[PC1]
            df_plot.multiple_series_scatter_chart('scatter plot', PC1, list(table.columns[1:n]), table)
        except Exception as e:
            print(e)
            print(traceback.format_exc())
            raise
        return table

    def categorical_column_encode(self, data: pd.DataFrame):
        columns = list(data.select_dtypes(include=['object']).columns)
        distinct_values = dict(map(lambda x: (x, len(pd.unique(data[x]))), columns))
        least_distinct_columns = list(dict(filter(lambda x: x[1] <= 5, distinct_values.items())).keys())
        mid_distinct_columns = list(dict(filter(lambda x: 5 < x[1] <= 10, distinct_values.items())).keys())
        most_distinct_values = list(dict(filter(lambda x: 10 < x[1], distinct_values.items())).keys())

        if len(least_distinct_columns):
            data = self.one_hot_encode(data, least_distinct_columns, drop_columns=True)
        if len(mid_distinct_columns):
            data = self.original_encode(data, mid_distinct_columns)
        if len(most_distinct_values):
            data = self.similarity_encode(data, most_distinct_values)

        return data

    def one_hot_encode(self, data: pd.DataFrame, columns, drop_columns):
        try:
            encoder = OneHotEncoder(sparse=False, handle_unknown='ignore').fit(data[columns])

            encoded_cols = list(encoder.get_feature_names_out(columns))

            data[encoded_cols] = encoder.transform(data[columns])
            if drop_columns:
                data = data.drop(columns, axis=1)

            for col in columns:
                print(col + " is " + "encoded")
            return data

        except Exception as e:
            raise type(e)(e)

    def original_encode(self, data: pd.DataFrame, columns):
        try:
            encoder = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=np.nan).fit(data[columns])
            new_names = [column + '_' + 'encode' for column in columns]
            data[new_names] = pd.DataFrame(encoder.transform(data[columns]))
            data.drop(columns, axis=1, inplace=True)
            for col in columns:
                print(col + " is " + "encoded")
            return data
        except Exception as e:
            raise type(e)(e)

    def similarity_encode(self, data: pd.DataFrame, columns):
        try:
            for col in columns:
                encoder = dirty_cat.SimilarityEncoder(similarity='ngram')
                encoded_df = pd.DataFrame(encoder.fit_transform(data[[col]]))
                encoded_df.columns = [col + '_' + str(i) for i in encoded_df.columns]
                data = pd.concat([data, encoded_df], axis=1)
                data.drop(col, axis=1, inplace=True)
                print(col + " is " + "encoded")

            return data
        except Exception as e:
            raise type(e)(e)

    def numeric_column_scale(self, data: pd.DataFrame):
        from sklearn.preprocessing import MinMaxScaler
        columns = list(data.select_dtypes(include=['int', 'float']).columns)
        for col in columns:
            if data[col].dtype == 'float':
                scaler = MinMaxScaler().fit(data[[col]])
                data[col] = scaler.transform(data[[col]])

                print(col + ' Is Normalized')
            else:
                if data[col].nunique() / len(data[col]) > 0.5:
                    scaler = MinMaxScaler().fit(data[[col]])
                    data[col] = scaler.transform(data[[col]]).round(2)
                    print(col + ' Is Normalized')
                else:
                    dummies = pd.get_dummies(data[col])
                    dummies.columns = [col + '_' + str(i) for i in list(dummies.columns)]
                    data = pd.concat([data, dummies], axis=1)
                    data = data.drop(col, axis=1)
                    print(col + ' Is Normalized')

        return data