import psycopg2
import json
import re
import unicodedata
from user_functions import VarIf, convertJSONDate, getKeyValue, QueryWithSingleValue, returnNextSerialID, cleanForSQL, checkRowExists
from database import cur, conn
from properties import Address
from contact_details import ContactDetails
from user_constants import OBJECT_Agent, OBJECT_Agency, FILE_TYPE_Images
from files import File

### Agency Class Definition
class Agency:

    variables_set = False

    def __init__( self, agency_banner, agency_website, agency_logo_standard, agency_long, agency_lat, agency_description, num_sale, num_rent, name, domain_agency_id,
                  principal_agent_name, raw_agency_json, contact_details, streetAddress1, streetAddress2, suburb, agents = None, principal_agent_id = None, agency_id = None,
                  state = None, postCode = None ):

        self.agency_banner = agency_banner
        self.agency_website = agency_website
        self.agency_logo_standard = agency_logo_standard
        cleaned_agency_description = self.cleanAgencyDescription( agency_description )
        #We need to truncate the agency description if its larger then 1000
        if len(cleaned_agency_description) > 1000:
            self.agency_description = cleaned_agency_description[0:997] + "..."
        else:
            self.agency_description = cleaned_agency_description
        self.num_sale = num_sale
        self.num_rent = num_rent
        self.domain_agency_id = domain_agency_id
        self.name = name
        self.principal_agent_name = principal_agent_name
        self.raw_agency_json = raw_agency_json
        self.agents = agents

        #Set up the address object for the Agency Class
        street_name = streetAddress1 + ' ' + streetAddress2
        full_address = street_name + ' ' + suburb + ' ' + state  + ' ' + str(postCode)
        #We need to get the street number
        street_number_object = re.search( r"\d+/?\d*", streetAddress1 )
        if street_number_object is None:
            street_number_object = re.search( r"\d+/?\d*", streetAddress2 )

        if street_number_object is not None:
            street_number = street_number_object.group(0)
        else:
            street_number = None

        self.address = Address( full_address, street_name, suburb, street_number, None, None, agency_long, agency_lat, state, postCode )

        self.contact_details = contact_details
        if principal_agent_id is not None:
            self.principal_agent_id = principal_agent_id
        else:
            self.principal_agent_id = None
        if agency_id is not None:
            self.agency_id = agency_id
        else:
            self.agency_id = None

        self.classesConverted = False

    def cleanAgencyDescription( self, cleanString ):
        #Use unicodedata.normalize to get rid of the extra Unicode Data
        agency_description_unicode = unicodedata.normalize( "NFKD", cleanString )
        remove_html_string = re.sub( r"\<\\?[^\>]+\>",
                                        "",
                                        agency_description_unicode )
        #str_length = len( agency_description_unicode )
        #final_step_string = None
        #count = 0
        #Find the position of the first span (We know that the main data will be contained inside a span)
        #first_span_position = agency_description_unicode.find( "<span" )
        #Slice the String and get the remainder.
        #first_step_string = agency_description_unicode[first_span_position:str_length]
        #Get the location of the >, signifiying the end of the first span tag.
        #end_first_span_position = first_step_string.find( ">" )
        #Slice the string again.
        #new_str_length = len(first_step_string)
        #second_step_string = first_step_string[end_first_span_position:new_str_length]
        #Now get the position for the closing span tag.
        #second_span_position = second_step_string.find("</span")
        #final_step_string = second_step_string[1:second_span_position]

        return remove_html_string

    def getPrincipalAgent( self ):
        try:
            #Test to see if the Agent instance has a value for principal_agent_name
            if self.principal_agent_name == None:
                raise ValueError( 'Invalid Principal Agent Name' )
            else:
                full_name = self.principal_agent_name.split()
                result = None
                if len(full_name) == 2 :
                    cur.execute( """ SELECT a.agent_id
                                  FROM agents a
                                  WHERE UPPER( a.first_name ) = UPPER( %s )
                                        AND UPPER( a.last_name ) = UPPER( %s )""",
                             ( cleanForSQL(full_name[0]), cleanForSQL(full_name[1] ) ) )
                    result = cur.fetchone()
                if result is None:
                    self.principal_agent_id = None
                else:
                    self.principal_agent_id = result[0]
            return True
        except( Exception, psycopg2.DatabaseError, ValueError ) as error:
            print( "Error in Getting Principal Agent ID for Agency " + self.name + "\n" + error )
            return False

    def hasAgencyID( self ):
        return self.agency_id is not None

    #def checkForNewAgents( self, agents ):
        #Need to first check if self.agents is populated and that the Agnecy Object has its agency id populated.
    #    if self.agents is None and self.agency_id is not None:
    #        cur.execute( """SELECT a.agent_id
    #                        FROM agencies_agent a
    #                        WHERE a.agency_id = %s
    #                            AND a.deleted_when IS NULL""",
    #                        ( self.agency_id ) )

    def convertToClasses( self ):
        """
        This Function converts all instance variables of the Agency Object that are meant to be classes...(i.e agency_banner) into their respective class.
        (These includes the Contact Details and Files Classes)

        """
        #Check to see if the instance has its agency_id set or has already had its variables converted into classes.
        try:
            if self.agency_id is None or self.classesConverted == True:
                raise Exception( "Cannot convert classes due to them already being converted or Agency Instance having a Null Agency ID" )
            else:
                agency_banner = File( FILE_TYPE_Images, OBJECT_Agency, self.agency_id, None, "agency_banner" )
                agency_banner.addImageDetails( None, None, self.agency_banner )
                self.agency_banner = agency_banner

                self.agency_website = ContactDetails( 'agency_website', OBJECT_Agency, self.agency_id, self.agency_website )

                agency_logo_standard = File( FILE_TYPE_Images, OBJECT_Agency, self.agency_id, None, "agency_logo_standard" )
                agency_logo_standard.addImageDetails( None, None, self.agency_logo_standard )
                self.agency_logo_standard = agency_logo_standard

                self.contact_details_converted = []
                for contactType, detailsType in self.contact_details.items():
                    #For now, only store dictionaries. We can handle exceptions by logging them to a debug file.
                    if isinstance( detailsType, dict ):
                        for contact, details in detailsType.items():
                                if details != '':
                                    self.contact_details_converted.append( ContactDetails( 'agency_' + contactType + '_' + contact, OBJECT_Agency, self.agency_id, details ) )
                    #else:
                    #    self.contact_details_converted.append( ContactDetails( 'agency_' + contactType + '_' + contact, self.agency_id, OBJECT_Agency, details ) )
                self.classesConverted = True

        except( Exception ) as error:
            print( error )

    def storeAgency( self ):
        try:
            #If the Agency ID hasn't been set for this Agency, it means that hasn't been saved yet into the database...
            if self.agency_id is None:
                self.agency_id = returnNextSerialID( 'agencies', 'agency_id' )

            if not self.getPrincipalAgent( ):
                return None
            #Convert the data inside the Agency object into Classes
            self.convertToClasses()

            #Save all contact details first.
            for contact in self.contact_details_converted:
                contact.storeContactDetails( False )

            #Store the links to the agency banner, website and standard logo.
            self.agency_website.storeContactDetails( False )
            self.agency_banner.storeFile( False )
            self.agency_logo_standard.storeFile( False )

            #Store the address first so that we can link the address to the correct row inside the agencies table.
            address_id = self.address.storeAddress()

            agency_json = json.dumps( self.raw_agency_json )

            cur.execute( """INSERT INTO agencies( description, num_sale, num_rent, entered_when, raw_agency_json, name, principal_agent, domain_agency_id, address_id )
                            VALUES( %s, %s, %s, current_timestamp, %s, %s, %s, %s, %s)""",
                            ( self.agency_description, self.num_sale, self.num_rent, cleanForSQL( agency_json ), self.name, self.principal_agent_id, self.domain_agency_id, address_id ) )
            return self.agency_id
        except( Exception, psycopg2.DatabaseError ) as error:
            print( "Error in INSERTING New Agency for Agency " + self.name + "\n" + error )
            return None

class Agent:
    def __init__( self, email, first_name, last_name, profile_text, mugshot_url, domain_agent_id, agency_id=None, facebook_url=None, twitter_url=None, phone=None, photo=None ):
        """
        This is the Init Function for the Agent Object.

        :param self: Agent Instance.
        :param email: Agent's email
        :param first_name: Agent's first name
        :param last_name: Agent's last name
        :param profile_text: Text representation of the agent's profile.
        :param mugshot_url: Url to the Agent's mugshot.
        :param domain_agent_id: ID of the Agent inside the Domain system.
        :param agency_id: The internal id of the Agent inside our database.
        :type agency_id: int (optional)
        :param facebook_url: Url to the Agent's FB page.
        :type facebook_url: str (optional)
        :param twitter_url: Url to the Agent's twitter page.
        :type twitter_url: str (optional)
        """
        self.email = email
        self.first_name = first_name
        self.phone = phone
        self.photo = photo
        self.last_name = last_name
        self.facebook_url = facebook_url
        self.twitter_url = twitter_url
        input_profile_text = self.cleanProfileText( profile_text )
        if input_profile_text is not None and len(input_profile_text) > 3000:
            self.profile_text = input_profile_text[0:2995] + "..."
        else:
            self.profile_text = input_profile_text
        self.mugshot_url = mugshot_url
        self.agency_id = agency_id
        self.domain_agent_id = domain_agent_id

    @classmethod
    def initFromObject( cls, agent_id ):
        #Alternate initialization method that queries the database for the specific agent_id and returns the relevant parameters to populate the Agent Instance.
        pass

    def cleanProfileText( self, profile_text ):
        if profile_text is None:
            return profile_text
        clean_ampersand = profile_text.replace( "&", "&" )
        clean_apostrophe = clean_ampersand.replace( "'", "'" )
        clean_rsquo = clean_apostrophe.replace("’", "'")
        clean_elipses = clean_rsquo.replace("…", "...")
        clean_quote = clean_rsquo.replace(""", "\"")
        clean_profile_text_remove_close_p = clean_quote.replace( "</p>", '\n' + '\n' )
        clean_profile_text = clean_profile_text_remove_close_p.replace( "<p>", "" )
        return clean_profile_text

    def storeAgent( self, commit ):
        try:
            self.agent_id = returnNextSerialID( 'agents', 'agent_id' )
            #Stores information contained inside the Agent Object into the Agents table.
            cur.execute( """INSERT INTO agents( domain_agent_id, entered_when, first_name, last_name, profile_text )
                            VALUES( %s, current_timestamp, %s, %s, %s )""" ,
                            ( self.domain_agent_id, cleanForSQL(self.first_name), cleanForSQL(self.last_name), self.profile_text ) )
            #Store the link between the Agent and the Agency inside the agencies_agent table.
            cur.execute( """INSERT INTO agencies_agent( agency_id, agent_id, entered_when )
                            VALUES( %s, %s, current_timestamp)""",
                            ( self.agency_id, self.agent_id ) )

            if self.email is not None:
                #Store the agent's emaia
                contactDetails_email = ContactDetails("agent_email", OBJECT_Agent, self.agent_id, self.email )
                if not contactDetails_email.storeContactDetails( False ):
                    raise Exception ( psycopg2.DatabaseError )

            if self.phone is not None:
                #Store the agent's phone number
                contactDetails_phone = ContactDetails("agent_phone_number", OBJECT_Agent, self.agent_id, self.phone )
                if not contactDetails_phone.storeContactDetails( False ):
                    raise Exception ( psycopg2.DatabaseError )

            if self.facebook_url is not None:

                contactDetails_facebook = ContactDetails( "agent_facebook_url", OBJECT_Agent, self.agent_id, self.facebook_url )
                #Attempt to save the facebook details
                if not contactDetails_facebook.storeContactDetails( False ):
                    raise Exception( psycopg2.DatabaseError )

            if self.twitter_url is not None:
                contactDetails_twitter = ContactDetails( "agent_twitter_url", OBJECT_Agent, self.agent_id, self.twitter_url )
                #Attempt to save the twitter details
                if not contactDetails_twitter.storeContactDetails( False ):
                    raise Exception( psycopg2.DatabaseError )

            if self.photo is not None:

                #Save the mugshot and agent photo.
                file_agentPhoto = File( FILE_TYPE_Images, OBJECT_Agent, self.agent_id, None, "agent_photo" )
                file_agentPhoto.addImageDetails( None, None, self.photo )
                if not file_agentPhoto.storeFile( False ):
                    raise Exception( psycopg2.DatabaseError )

            if self.mugshot_url is not None:

                file_agentMugShot = File( FILE_TYPE_Images, OBJECT_Agent, self.agent_id, None, "agent_mugshot" )
                file_agentMugShot.addImageDetails( None, None, self.mugshot_url )
                if not file_agentMugShot.storeFile( False ):
                    raise Exception( psycopg2.DatabaseError )

            if commit:
                conn.commit()

            return self.agent_id

        except(Exception, psycopg2.DatabaseError) as error:
            print("Error in INSERTING New Agent " + self.first_name + " " + self.last_name + "\n" + error )
            return None


#! /usr/bin/env python
import click
import os
import logging

logger = logging.getLogger(__name__)

@click.group()
@click.version_option()
@click.option('--debug', 'debug_mode', is_flag=True)
def cli(debug_mode):
    """
    A toolset for investigating the interactions between circRNA, miRNA, and mRNA.
    """

    root_logger = logging.getLogger()

    ch = logging.StreamHandler()

    if debug_mode:
        logger.warning('Under debug mode')
        root_logger.setLevel(logging.DEBUG)
        formatter = logging.Formatter('%(asctime)s (%(name)s) [%(levelname)s] %(message)s')
    else:
        root_logger.setLevel(logging.INFO)
        formatter = logging.Formatter('%(asctime)s - %(message)s')

    ch.setFormatter(formatter)
    root_logger.addHandler(ch)

@cli.command('interactions')
@click.option('-r', '--ref', 'ref_dir', type=click.Path(), metavar="REF_DIR", required=True)
@click.option('-i', '--circ', 'circ_file', metavar="CIRC_FILE", required=True)
@click.option('-o', '--out-prefix', 'out_prefix', default='./', metavar="OUT_PREFIX")
@click.option('-p', '--num_proc', default=1, type=click.INT, metavar="NUM_PROC",
    help="Number of processes")
@click.option('--checkAA', 'checkAA', is_flag=True,
    help="Check if the circRNA has ambiguous alignments.", hidden=True)
@click.option('--no-pvalue-filtering', 'pv_filter', flag_value=False, default=True,
    help="If this option is set, the results will contain all interactions without P-values filtering.")
@click.option('--miranda-sc', 'sc', metavar='S', type=click.FLOAT, default=155, help='(Default: 155)')
@click.option('--miranda-en', 'en', metavar='-E', type=click.FLOAT, default=-20, help='(Default: -20)')
@click.option('--miranda-scale', 'scale', metavar='Z', type=click.FLOAT)
@click.option('--miranda-strict', 'strict', is_flag=True)
@click.option('--miranda-go', 'go', metavar='-X', type=click.FLOAT)
@click.option('--miranda-ge', 'ge', metavar='-Y', type=click.FLOAT)
def predict_interactions(circ_file,
                         ref_dir,
                         out_prefix,
                         num_proc,
                         checkAA,
                         pv_filter,
                         **miranda_options):

    """
    Predict the interactions.

    Predict the interactions between circRNA-miRNA-mRNA.
    """

    logger.info('Preparing ...')

    from circmimi.utils import add_prefix

    output_dir = os.path.dirname(out_prefix)
    if output_dir == '':
        output_dir = '.'

    if output_dir != '.':
        os.makedirs(output_dir, exist_ok=True)

    from circmimi.reference.config import get_refs
    anno_db, ref_file, mir_ref, mir_target, other_transcripts, AGO_data, RBP_data, RBP_target = get_refs(ref_dir)

    if checkAA:
        other_ref_file = other_transcripts
    else:
        other_ref_file = None

    miranda_options_list = []
    for k, v in miranda_options.items():
        if v is not None:
            cmd = ['-' + k]
            if k != 'strict':
                cmd.append(str(v))

            miranda_options_list.extend(cmd)

    from circmimi.circmimi import Circmimi
    circmimi_result = Circmimi(
        anno_db,
        ref_file,
        mir_ref,
        mir_target,
        AGO_data,
        RBP_data,
        RBP_target,
        other_ref_file,
        work_dir=output_dir,
        num_proc=num_proc,
        pv_filter=pv_filter,
        miranda_options=miranda_options_list
    )

    logger.info('Starting the main pipeline.')
    circmimi_result.run(circ_file)
    logger.info('Pipeline completed.')

    logger.info('Saving results ...')
    res_file = add_prefix('all_interactions.miRNA.tsv', out_prefix)
    circmimi_result.save_result(res_file)

    circ_target_pv_file = add_prefix('circRNA_target_gene.pvalue.tsv', out_prefix)
    circmimi_result.circ_target_df_with_pv.to_csv(circ_target_pv_file, sep='\t', index=False)

    logger.info('miRNA part ... done')

    # RBP_res_file = add_prefix('all_interactions.RBP.tsv', out_prefix)
    # circmimi_result.save_RBP_result(RBP_res_file)
    # logger.info('RBP part ... done')

    summary_file = add_prefix('summary_list.tsv', out_prefix)
    circmimi_result.save_circRNAs_summary(summary_file)
    logger.info('summary file ... done')
    logger.info('All results are saved.')

    logger.info('Process completed.')

@cli.command('genref')
@click.option('--species', 'species', metavar="SPECIES_KEY", required=True, help="eg. hsa, mmu, rno, ...")
@click.option('--source', 'source', metavar="SOURCE", required=True,
    help="""
        Available sources are "gencode", "ensembl", "ensembl_plants", and "ensembl_metazoa"
    """)
@click.option('--gencode', 'source', flag_value='gencode', type=click.STRING, help="--source gencode", hidden=True)
@click.option('--ensembl', 'source', flag_value='ensembl', type=click.STRING, help="--source ensembl", hidden=True)
@click.option('--version', 'version', default='current', metavar="VERSION",
    help="""
        The release version. If not specified, it will be automatically set to the latest version of the SOURCE.
    """)
@click.option('--init', 'init', is_flag=True, help="Create an init template ref_dir.", hidden=True)
@click.argument('ref_dir')
def generate_references(species, source, version, ref_dir, init):
    """
    Generate the references.

    This command is used to generate the needed references for CircMiMi.
    It would download the genome ane annotation files automatically, with user-specified version.

    The generated reference files will be saved in the REF_DIR.
    """

    os.makedirs(ref_dir, exist_ok=True)

    from circmimi.reference.config import RefConfig
    config = RefConfig()

    if not init:
        from circmimi.reference import genref
        info, ref_files = genref.generate(species, source, version, ref_dir)

        config['info'].update(info)
        config['refs'].update(ref_files)

    config.write(ref_dir)

@cli.command('gendb', hidden=True)
@click.argument('gtf_path')
@click.argument('db_path', metavar='OUT_PATH')
def generate_annotation_database(gtf_path, db_path):
    from circmimi.reference import gendb

    gendb.generate(gtf_path, db_path)

@cli.command('genmirdb', hidden=True)
@click.option('--species', 'species', metavar="SPECIES_KEY", required=True)
@click.option('--version', 'version', default='current', metavar="VERSION", required=True)
@click.option('-r', '--ref', 'ref_dir', type=click.Path(), metavar="REF_DIR", required=True)
@click.option('-o', '--out_file', 'out_file', metavar="OUT_FILE", required=True)
@click.option('-a', '--show-accession', 'show_accession', is_flag=True)
def generate_miRNA_database(species, version, ref_dir, out_file, show_accession):
    os.makedirs(ref_dir, exist_ok=True)

    from circmimi.reference import genmirdb

    genmirdb.generate(species, version, ref_dir, out_file, show_accession)

@cli.group(hidden=True)
def check():
    pass

@check.command('annotation')
@click.argument('anno_db')
@click.argument('circ_file')
@click.argument('out_file')
def check_annotation(circ_file, anno_db, out_file):
    from circmimi.circ import CircEvents

    circ_events = CircEvents(circ_file)
    circ_events.check_annotation(anno_db)

    circ_events.get_filters_results(True).to_csv(out_file, sep='\t', index=False)

@check.command('ambiguous')
@click.argument('ref_file')
@click.argument('other_ref_file')
@click.argument('circ_file')
@click.argument('out_file')
@click.option('-p', '--num_proc', default=1, type=click.INT,
              metavar="NUM_PROC", help="Number of processes")
def check_ambiguous(ref_file, other_ref_file, circ_file, out_file, num_proc):
    import subprocess as sp
    import tempfile as tp

    tmp_fa = tp.NamedTemporaryFile(
        dir='.',
        prefix='flanking_seq.',
        suffix='.fa'
    )

    sp.run(['get_flanking_seq.py', ref_file, circ_file, tmp_fa.name])
    sp.run(['checkAA_reads.py', '-rg', ref_file, '-ro', other_ref_file,
            tmp_fa.name, out_file, '-p', str(num_proc)])

@check.command('RCS')
@click.argument('ref_file')
@click.argument('circ_file')
@click.argument('out_file')
@click.option('-d', '--dist', default=10000, type=click.INT)
@click.option('-m', '--min-matches', default=80, type=click.FLOAT)
@click.option('-l', '--min-aln-len', default=50, type=click.INT)
@click.option('-b', '--min-bitscore', default=100, type=click.FLOAT)
@click.option('-p', '--num_proc', default=1, type=click.INT,
              metavar="NUM_PROC", help="Number of processes")
def check_RCS(ref_file,
              circ_file,
              out_file,
              dist,
              min_matches,
              min_aln_len,
              min_bitscore,
              num_proc):

    import subprocess as sp

    cmd1 = ['get_RCS.py', ref_file, circ_file, '--dist', dist, '-p', num_proc]
    cmd2 = ['RCS_filter.py', '-', '-m', min_matches, '-l', min_aln_len, '-b', min_bitscore]
    cmd3 = ['get_RCS_summary.py', '--circ_file', circ_file, '-']

    cmd1 = [str(c) for c in cmd1]
    cmd2 = [str(c) for c in cmd2]
    cmd3 = [str(c) for c in cmd3]

    with open(out_file, 'w') as out:
        p1 = sp.Popen(cmd1, stdout=sp.PIPE, encoding='UTF-8')
        p2 = sp.Popen(cmd2, stdin=p1.stdout, stdout=sp.PIPE, encoding='UTF-8')
        p3 = sp.Popen(cmd3, stdin=p2.stdout, stdout=out, encoding='UTF-8')

        p1.wait()
        p2.wait()
        p3.wait()

@cli.command('checking')
@click.option('-i', '--circ', 'circ_file', metavar="CIRC_FILE", required=True)
@click.option('-r', '--ref', 'ref_dir', type=click.Path(), metavar="REF_DIR",
              required=True)
@click.option('-o', '--out-prefix', 'out_prefix', default='./',
              metavar="OUT_PREFIX")
@click.option('-p', '--num_proc', default=1, type=click.INT,
              metavar="NUM_PROC", help="Number of processes")
@click.option('-d', '--dist', default=10000, type=click.INT)
@click.pass_context
def default_checking(ctx, circ_file, ref_dir, out_prefix, num_proc, dist):
    """
    Check the circRNAs.

    Check the circRNAs if there are
    (i) ambiguous alignments for the fragment around the circRNA junction;
    (ii) RCS pair in the flanking sequences.
    """

    import tempfile as tp
    from circmimi.reference.config import get_refs
    from circmimi.utils import add_prefix

    output_dir = os.path.dirname(out_prefix)
    if output_dir == '':
        output_dir = '.'

    if output_dir != '.':
        os.makedirs(output_dir, exist_ok=True)

    anno_db, ref_file, _, _, other_transcripts, _, _, _ = get_refs(ref_dir)

    annotation_result_file = tp.NamedTemporaryFile(
        dir=output_dir,
        prefix='check.annotation.',
        suffix='.tsv'
    )

    ambiguous_result_file = tp.NamedTemporaryFile(
        dir=output_dir,
        prefix='check.AA.',
        suffix='.tsv'
    )
    RCS_result_file = tp.NamedTemporaryFile(
        dir=output_dir,
        prefix='check.RCS.',
        suffix='.tsv'
    )
    out_file = add_prefix('checking.results.tsv', out_prefix)

    ctx.invoke(
        check_annotation,
        circ_file=circ_file,
        anno_db=anno_db,
        out_file=annotation_result_file.name
    )

    ctx.invoke(
        check_ambiguous,
        ref_file=ref_file,
        other_ref_file=other_transcripts,
        circ_file=circ_file,
        out_file=ambiguous_result_file.name,
        num_proc=num_proc
    )

    ctx.invoke(
        check_RCS,
        ref_file=ref_file,
        circ_file=circ_file,
        out_file=RCS_result_file.name,
        num_proc=num_proc,
        dist=dist
    )

    import pandas as pd

    ann_df = pd.read_csv(annotation_result_file.name, sep='\t', dtype='object')
    amb_df = pd.read_csv(ambiguous_result_file.name, sep='\t', dtype='object')
    rcs_df = pd.read_csv(RCS_result_file.name, sep='\t', dtype='object')
    result_df = pd.concat(
        [
            ann_df,
            amb_df.drop('circRNA_id', axis=1),
            rcs_df.drop('circRNA_id', axis=1)
        ],
        axis=1
    )
    result_df.to_csv(out_file, sep='\t', index=False)

@cli.group(hidden=True)
def network():
    """
    Command to create the network file for Cytoscape.
    """

    pass

@network.command('create')
@click.argument('in_file')
@click.argument('out_file')
@click.option('-1', 'idx_circRNA', type=int, default=1,
              help='column key for circRNAs.')
@click.option('-2', 'idx_mediator', type=int, default=2,
              help='column key for mediators.')
@click.option('-3', 'idx_mRNA', type=int, default=3,
              help='column key for mRNAs.')
@click.option('-f', '--format', 'format_', default='xgmml',
              help="Assign the format of the OUT_FILE.", hidden=True)
def create_network(in_file, out_file, idx_circRNA, idx_mediator, idx_mRNA, format_):
    """
    Create the network file.

    This command would transform the IN_FILE to XGMML format,
    and save to OUT_FILE.
    """

    from circmimi.network.network import CyNetwork, Layout, Style

    network = CyNetwork()
    network.load_data(
        in_file,
        k1=idx_circRNA,
        k2=idx_mediator,
        k3=idx_mRNA
    )

    layout = Layout()
    style = Style()

    network.apply_layout(layout)
    network.apply_style(style)

    if format_ == "xgmml":
        network.to_xgmml(out_file)

@cli.command('visualize')
@click.argument('in_file')
@click.argument('out_file')
@click.option('-1', 'idx_circRNA', type=int, default=5,
              help='column key for circRNAs.')
@click.option('-2', 'idx_mediator', type=int, default=7,
              help='column key for mediators.')
@click.option('-3', 'idx_mRNA', type=int, default=13,
              help='column key for mRNAs.')
@click.option('-f', '--format', 'format_', default='xgmml',
              help="Assign the format of the OUT_FILE.", hidden=True)
@click.option('--header', 'header', flag_value=True, type=click.BOOL,
              default=True, hidden=True,
              help="Use this option to specify if there are titles in the IN_FILE.")
@click.option('--no-header', 'header', flag_value=False, type=click.BOOL, default=True,
              help="Use this option to specify if there are no titles in the IN_FILE.")
def visualize_interactions(in_file,
                           out_file,
                           idx_circRNA,
                           idx_mediator,
                           idx_mRNA,
                           header,
                           format_):

    """
    Visualize the interactions.

    This command is used to convert the file of interactions into XGMML format,
     which then can be loaded into Cytoscape for visualization and further analysis.
    """

    from circmimi.network.network import CyNetwork, Layout, Style

    network = CyNetwork()
    network.load_data(
        in_file,
        k1=idx_circRNA,
        k2=idx_mediator,
        k3=idx_mRNA,
        header=header
    )

    layout = Layout()
    style = Style()

    network.apply_layout(layout)
    network.apply_style(style)

    if format_ == "xgmml":
        network.to_xgmml(out_file)

@cli.group('update_mirna', hidden=True)
def update_mirna():
    pass

@update_mirna.command('genmaps')
@click.option('--from', 'from_', required=True)
@click.option('--to', 'to_', required=True)
@click.option('--species')
@click.option('-o', '--out-prefix', 'out_prefix', default='./', metavar="OUT_PREFIX")
def generate_miRNAs_mappings(from_, to_, species, out_prefix):
    from circmimi.reference.mirbase import MatureMiRNAUpdater
    from circmimi.utils import add_prefix

    output_dir, prefix_name = os.path.split(out_prefix)

    if output_dir == '':
        output_dir = '.'

    if output_dir != '.':
        os.makedirs(output_dir, exist_ok=True)

    updater = MatureMiRNAUpdater(from_, to_, species)
    updater.create(output_dir)

    out_file = 'miRNA.maps_{}_to_{}.tsv'.format(from_, to_)
    out_file = add_prefix(out_file, prefix_name)
    out_file = os.path.join(output_dir, out_file)

    updater.save(out_file)

@update_mirna.command('update')
@click.argument('in_file')
@click.argument('out_file')
@click.option('-m', '--maps', 'mapping_file', metavar="MAPPING_FILE", required=True)
@click.option('-k', '--col-key', 'column_key', type=click.INT, default=1,
              help="The column number of miRNA IDs.")
@click.option('-i', '--inplace', is_flag=True)
@click.option('-R', '--remove-deleted', is_flag=True)
def update_miRNAs(in_file, out_file, mapping_file, column_key, inplace, remove_deleted):
    column_key = column_key - 1

    from circmimi.reference.mirbase import MatureMiRNAUpdater
    updater = MatureMiRNAUpdater(None, None, None)
    updater.load_maps(mapping_file)
    updater.update_file(in_file, out_file, column_key, inplace, remove_deleted)

@cli.command('calc-pvalue', hidden=True)
@click.argument('interaction_file')
@click.option('--mir_ref_file')
@click.option('--mir_target_file')
@click.option('-o', '--out_prefix', default='./')
def calculate_pvalue_of_interactions(interaction_file, out_prefix, mir_ref_file, mir_target_file):
    """
    This command is used to calculate the P-value of the interaction between circRNA and target gene.
    """

    import pandas as pd
    from circmimi.stats import do_the_hypergeometric_test
    from circmimi.circmimi import get_mir_target_db

    df = pd.read_csv(interaction_file, sep='\t', dtype='object')
    circ_mir_target_df = df[['circ_id', 'mirna', 'target_gene']].drop_duplicates().reset_index(drop=True)

    mir_target_db = get_mir_target_db(mir_target_file)

    circ_target_df_with_pv = do_the_hypergeometric_test(
        circ_mir_target_df,
        mir_ref_file,
        mir_target_db
    )

    circ_target_df_with_pv.to_csv(out_prefix + "circRNA_target_gene.pvalue.tsv", sep='\t', index=False)

    result_df = df.merge(
        circ_target_df_with_pv[[
            'circ_id',
            'target_gene',
            'p_value',
            'bh_corrected_p_value',
            'bonferroni_corrected_p_values',
        ]],
        on=['circ_id', 'target_gene'],
        how='left'
    )

    result_df.to_csv(out_prefix + "all_interactions.miRNA.pvalue.tsv", sep='\t', index=False)

if __name__ == "__main__":
    cli()


import colorsys

# █▀▄▀█ █▀▀ ▀█▀ ▄▀█ ▀
# █░▀░█ ██▄ ░█░ █▀█ ▄
# -------------------
__author__ = "MOIS3Y"
__credits__ = ["Stepan Zhukovsky"]
__license__ = "GPL v3.0"
__version__ = "0.2.0"
__maintainer__ = "Stepan Zhukovsky"
__email__ = "stepan@zhukovsky.me"
__status__ = "Production"

# █▀▀ █▀█ █░░ █▀█ █▀█ ▀
# █▄▄ █▄█ █▄▄ █▄█ █▀▄ ▄
# -------------------
class Color(object):
    """This class is intended to be stored in an attribute
    Color.scheme color scheme in base16 format
    https://github.com/chriskempson/base16
    And its subsequent use to set the color for various elements.
    Since there are only 16 colors in base16, sometimes this is not enough to
    place color accents anywhere.
    Therefore, the class has several methods for adjusting the color.
    The HSL color model is used here.
        HLS: Hue, Luminance, Saturation
            H: position in the spectrum
            S: color saturation
            L: color lightness
    You are probably familiar with it, if not, it's best to just see how
    you can change the color by changing one of the parameters,
    for example here:
    https://hslpicker.com/
    Each channel has its own method which accepts a color and a ratio
    offsets:
        color.hue(color, shift)
        color saturation(color, shift)
        Color.lightness(color, shift)

    For saturation, lightness, the shift argument takes the offset percentage
    relative to the current color (-100, 100), if you pass a percentage
    that will go beyond frame, then the parameter being changed will take
    the maximum or minimum possible value.
    For hue, the absolute value of the degree that H can
    be changed this channel (-360, 360).

        Examples:
        ---------

        color_scheme = some_base16_theme

        color = Color(**color_scheme)

        shift_color = color.scheme['base08']                =>  #b4befe

        hue_color = Color.hue(shift_color, -60)             =>  #b4fef4

        saturation = Color.saturation(shift_color, 20)      =>  #b3bdff

        lightness_color = Color.lightness(shift_color, 20)  =>  #e6eaff
    """
    def __init__(self, **color_scheme) -> dict:
        """Stores the color scheme as an attribute of the scheme class
        preferably in base16 simple example:
        themes = {
            #....
            "catppuccin_mocha": {
                "scheme": "Catppuccin Mocha",
                "author": "https://github.com/catppuccin/catppuccin",
                "base00": "#11111b", # crust
                "base01": "#181825", # mantle
                "base02": "#313244", # surface0
                "base03": "#45475a", # surface1
                "base04": "#585b70", # surface2
                "base05": "#cdd6f4", # text
                "base06": "#f5e0dc", # rosewater
                "base07": "#b4befe", # lavender
                "base08": "#f38ba8", # red
                "base09": "#fab387", # peach
                "base0A": "#f9e2af", # yellow
                "base0B": "#a6e3a1", # green
                "base0C": "#94e2d5", # teal
                "base0D": "#89b4fa", # blue
                "base0E": "#cba6f7", # mauve
                "base0F": "#f2cdcd", # flamingo
            },
            #....
        }
        color_scheme = Color(themes['catppuccin_mocha'])
        """
        self.scheme = color_scheme

    def hex_to_rgb(self, color: str) -> tuple:
        """
        Args:
            color (_str_): '#b4befe'
        Returns:       R    G    B
            _tuple_: (180, 190, 254)
        """
        hex_value = color.strip('#')
        rgb = tuple(int(hex_value[item:item + 2], 16) for item in (0, 2, 4))
        return rgb  # (RGB)

    def rgb_to_hex(self, *rgb) -> str:
        """
        Args:                 R    G    B
            *rgb (_tuple_): (180, 190, 254)
        Returns:
            _str_: '#b4befe'
        """
        r, g, b = [int(channel) for channel in rgb]
        return '#{:02x}{:02x}{:02x}'.format(r, g, b)  # #HEX

    def hex_to_hsl(self, color: str) -> tuple:
        """
        Args:
            color (_str_): '#b4befe'
        Returns:       H    S    L
            _tuple_: (0.64 0.97 0.85)
            16 decimal places, rounding is not performed to preserve precision.
        """
        rgb = self.hex_to_rgb(color)
        return (self.rgb_to_hsl(*rgb))

    def hsl_to_hex(self, *hsl) -> str:
        """
        Args:                  H    S    L
            *hsl (_tuple_): (0.64 0.97 0.85)
        Returns:
            _str_: '#b4befe'
        """
        rgb = self.hsl_to_rgb(*hsl)
        return (self.rgb_to_hex(*rgb))  # #HEX

    def rgb_to_hsl(self, *rgb) -> tuple:
        """
        Args:                 R    G    B
            *rgb (_tuple_): (180, 190, 254)
        Returns:
            _tuple_: (0.64 0.97 0.85)
            16 decimal places, rounding is not performed to preserve precision.
        """
        # Normalize R, G, B values:
        r, g, b = [item / 255.0 for item in rgb]
        h, l, s = [item for item in colorsys.rgb_to_hls(r, g, b)]  # noqa: E741
        return (h, s, l)

    def hsl_to_rgb(self, *hsl) -> tuple:
        """
        Args:                  H    S    L
            *hsl (_tuple_): (0.64 0.97 0.85)
        Returns:       R    G    B
            _tuple_: (180, 190, 254)
        """
        h, s, l = [channel for channel in hsl]   # noqa: E741
        r, g, b = [round(item * 255) for item in colorsys.hls_to_rgb(h, l, s)]
        return (r, g, b)

    @staticmethod
    def _check_range(channel: float) -> int | float:
        return 0 if channel < 0 else (1 if channel > 1 else channel)

    def hue(self, color: str, shift: int) -> str:
        """position in the spectrum 0 -> 360 degrees
        takes an int number and shift the current position by that number:
        to increase, the number must be positive (10)
        to decrease, the number must be negative (-10)

        Args:
            color (_str_): '#b4befe'
            shift (_int_): 10 or -10 or 42 or -42 etc
        Returns:
            _str_: #feb4b4
        """
        h, s, l = [channel for channel in self.hex_to_hsl(color)]  # noqa: E741
        if shift < 0:
            h = self._check_range(((h * 3.6 - abs(shift) / 100) / 3.6))
        else:
            h = self._check_range(((h * 3.6 + shift / 100) / 3.6))
        return self.hsl_to_hex(h, s, l)

    def saturation(self, color: str, shift: int) -> str:
        """color saturation 0 -> 100%
        takes an int (percentage) and shifts the current position
        by that percentage:
        to increase, the number must be positive (20)
        to decrease, the number must be negative (-20)

        Args:
            color (_str_): '#b4befe'
            shift (_int_): 10 or -10 or 42 or -42 etc
        Returns:
            _str_: '#b3bdff'
        """
        h, s, l = [channel for channel in self.hex_to_hsl(color)]  # noqa: E741
        if shift < 0:
            s = self._check_range((s - (abs(shift) / 100)))
        else:
            s = self._check_range((s + (shift / 100)))
        return self.hsl_to_hex(h, s, l)

    def lightness(self, color: str, shift: int) -> str:
        """color lightness 0 -> 100%
        takes an int (percentage) and shifts the current position
        by that percentage:
        to increase, the number must be positive (20)
        to decrease, the number must be negative (-20)

        Args:
            color (_str_): '#b4befe'
            shift (_int_): 10 or -10 or 42 or -42 etc
        Returns:
            _str_: '#b3bdff'
        """
        h, s, l = [channel for channel in self.hex_to_hsl(color)]  # noqa: E741
        if shift < 0:
            l = self._check_range((l - (abs(shift) / 100)))  # noqa: E741
        else:
            l = self._check_range((l + (shift / 100)))  # noqa: E741
        return self.hsl_to_hex(h, s, l)


"""
Part1 solution

author Yajue Yang
"""

from PA_1.data_processing import load_polydata
from PA_1.data_processing import feature_poly_tf
from PA_1.regression_alg import *
import random
from PA_1.result_analysis import *

def err_vs_training_size():
    """
    Find the error versus training size
    :return:
    """
    ratio = [1, 0.85, 0.75, 0.6, 0.5, 0.25, 0.15]

    mse_stat = np.ndarray((5, len(ratio)))

    num_trials = 10

    for alg_idx in range(5):

        for r_idx in range(len(ratio)):

            mse = 0
            theta = 0
            mu_e = 0

            for trial_idx in range(num_trials):
                sample_x, sample_y = load_polydata(ratio[r_idx], shuffle=True)
                test_x, test_y = load_polydata(1, shuffle=False)
                sample_phi = feature_poly_tf(sample_x, 5)
                test_phi = feature_poly_tf(test_x, 5)
                if alg_idx == 0:
                    theta = LS_regression(sample_phi, np.array(sample_y))
                elif alg_idx == 1:
                    theta = RLS_regression(sample_phi, np.array(sample_y), 5)
                elif alg_idx == 2:
                    theta = LASSO_regression(sample_phi, np.array(sample_y), 5)
                elif alg_idx == 3:
                    theta = RR_regression(sample_phi, np.array(sample_y))
                else:
                    mu_e, var_e = BR_regression(sample_phi, sample_y, 10)

                if alg_idx == 4:
                    et_y = np.transpose(test_phi) @ mu_e
                else:
                    et_y = np.transpose(test_phi) @ theta

                mse += mean_square_err(et_y, test_y)

            mse_stat[alg_idx, r_idx] = mse / num_trials

        print(alg_idx, r_idx, mse_stat[alg_idx, r_idx])

    # plot
    fig, axes = plt.subplots(5)

    colors = ['r', 'g', 'b', 'y', 'm']
    labels = ['LS', 'RLS', 'LASSO', 'RR', 'BR']
    for i in range(5):
        axes[i].plot(ratio, mse_stat[i, :], colors[i], label=labels[i])
        axes[i].legend()

    fig.text(0.5, 0.04, "training size", ha="center", va="center")
    fig.text(0.05, 0.5, "mean-squared error", ha="center", va="center", rotation=90)

    plt.show()

    for i in range(5):
        plt.plot(ratio, mse_stat[i, :], colors[i], label=labels[i])

    plt.xlabel('training size')
    plt.ylabel('mean-squared error')

    plt.legend()
    plt.show()

def add_outliers(s_y, ratio=0.1, mu=5, std=5):
    """
    Add outliers output values with Gaussian noise
    :param s_y: sampley
    :param ratio: the size of values to be modified
    :return: modified y
    """
    num_sample = len(s_y)
    num_modified = int(num_sample * ratio)

    print('number of modified data: ', num_modified)

    indices_modified = random.sample(range(num_sample), num_modified)

    print('modified indices: ', indices_modified)

    modified_y = s_y.copy()

    for i in indices_modified:
        noise = np.random.normal(mu, std, 1)
        # print('noise: ', noise)
        modified_y[i] += noise
        # print(s_y[i], modified_y[i])

    return modified_y

def compare_algorithms(sample_x, sample_y, test_x, test_y, order):

    # 5th order polynomial inputs
    sample_phi = feature_poly_tf(sample_x, order)
    test_phi = feature_poly_tf(test_x, order)

    theta = 0
    mu_e = 0
    var_e = 0

    algo_name = ['Least Squares', 'Regularized Least Squares', 'LASSO', 'Robust Regression', 'Bayesian Regression']

    for i in range(5):
        if i == 0:
            theta = LS_regression(sample_phi, np.array(sample_y))
        elif i == 1:
            theta = RLS_regression(sample_phi, np.array(sample_y), 5)
        elif i == 2:
            theta = LASSO_regression(sample_phi, np.array(sample_y), 5)
        elif i == 3:
            theta = RR_regression(sample_phi, np.array(sample_y))
        else:
            mu_e, var_e = BR_regression(sample_phi, sample_y, 10)

        if i == 4:
            et_y = np.transpose(test_phi) @ mu_e
            var_star = np.transpose(test_phi) @ var_e @ test_phi
            std_err = np.sqrt(var_star.diagonal())
            br_func_plot(test_x, et_y, std_err, sample_x, sample_y)
        else:
            et_y = np.transpose(test_phi) @ theta
            estimated_func_plot(test_x, et_y, sample_x, sample_y, algo_name[i])

        print(algo_name[i] + ' mse: ', mean_square_err(et_y, test_y))

def outlier_test():

    sample_x, sample_y = load_polydata()
    test_x, test_y = load_polydata(sampled=False)
    sample_y = add_outliers(sample_y, ratio=0.1, mu=10, std=10)

    compare_algorithms(sample_x, sample_y, test_x, test_y, 5)

def ten_order_test():
    sample_x, sample_y = load_polydata()
    test_x, test_y = load_polydata(sampled=False)

    compare_algorithms(sample_x, sample_y, test_x, test_y, 10)

if __name__ == '__main__':

    """
    part1-c
    """
    # err_vs_training_size()

    """
    part1-d
    """
    # outlier_test()

    """
    part1-e
    """
    ten_order_test()


import pandas as pd
import sklearn
import sklearn.datasets
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from time import time

from model import *

def alpha_decay(a, iteration):
  if iteration % 500 == 0:
    return a * 0.95
  return a

def sa_uniform_sampler(X_train, X_test, Y_train, Y_test, minval=-1, maxval=1):
  print("\n\t Evaluating IrisNet with Uniform Sampler")
  net = IrisNet()
  t1 = time()
  net.fit(X_train, X_test, Y_train, Y_test,
          T0=20000.0, alpha=4.5,
          alpha_decay=alpha_decay,
          T_decay=lambda T, a: T - a,
          sampler=UniformSampler(minval, maxval))
  t2 = time() - t1
  print(f"SA with uniform sampling took {t2:.3f} seconds")

def sa_gaussian_sampler(X_train, X_test, Y_train, Y_test, mu=0, std=3):
  print("\n\t Evaluating IrisNet with Gaussian Sampler")
  net = IrisNet()
  t1 = time()
  net.fit(X_train, X_test, Y_train, Y_test,
          T0=4000.0, alpha=1.5,
          alpha_decay=alpha_decay,
          T_decay=lambda T, a: T - a,
          sampler=GaussianSampler(mu, std))
  t2 = time() - t1
  print(f"SA with gaussian sampling took {t2:.3f} seconds")

def iris_sgd(X_train, X_test, Y_train, Y_test):
  print("\n\t Evaluating IrisNet with SGD")
  net = IrisNet()
  t1 = time()
  net.fit_sgd(X_train, X_test, Y_train, Y_test)
  t2 = time() - t1
  print(f"Adam took {t2:.3f} seconds")

def main():
  iris = sklearn.datasets.load_iris()
  X, Y = iris.data, iris.target

  scaler = preprocessing.StandardScaler()

  X = pd.DataFrame(scaler.fit_transform(X))
  Y = pd.DataFrame(Y.reshape(-1))
  X_train, X_test, Y_train, Y_test = train_test_split(X, Y)

  sa_uniform_sampler(X_train, X_test, Y_train, Y_test)
  sa_gaussian_sampler(X_train, X_test, Y_train, Y_test)
  iris_sgd(X_train, X_test, Y_train, Y_test)

if __name__ == '__main__':
  main()


# support
import os
import sys
import pathlib
from time import sleep
from serenityapp.lang import L
import ttkbootstrap as ttk
from typing import TypeVar
from serenityapp.themes import USER_THEMES

# useful functions

def is_past_list(index, list) -> bool:
    return index > len(list) - 1

def is_last_index(index, list) -> bool:
    return index == len(list) - 1

def next_valid_index(from_index: int, circular_list: list) -> int:
    if from_index < len(circular_list) - 1:
        return from_index + 1
    else:
        return 0

def path_exists(path):
    return os.path.exists(os.path.expanduser(path))

def light_wait():
    sleep(0.001)

def is_netpath(path):
    return '://' in path[0:13]

# typing
PathLike = TypeVar("PathLike", str, pathlib.Path)

SUPPORTED_EXTENSIONS = (
    ".mp3", ".wav", ".aac",
    ".wma", ".flac", ".ogg",
    ".opus", ".m4a", ".wma",
    ".pcm", ".au", ".3gp",
)  # THIS IS FAR FROM EXHAUSTIVE

DOWNLOADS_CODEC = 'mp3'

ART_FORMAT = 'jpg'

# only splitterchar should be changed by the user
SPLITTER_CHAR = '-'
SPLITTER = " " + SPLITTER_CHAR + " "

# placeholders
UNKNOWN_ARTIST = L['UNKNOWN_ARTIST']
UNKNOWN_ALBUM = "---"
UNKNOWN_ALBUM_ARTIST = "---"

# directories
# path to config w/ slash appended
if getattr(sys, 'frozen', False):
    DIR_PATH = os.path.dirname(sys.executable) + os.sep
else:
    DIR_PATH = os.getcwd() + os.sep
# should change to os.cwd() + "art" + os.sep in future
ART_PATH = DIR_PATH + "art" + os.sep  # TODO: remove

PLAYLIST_FOLDER_NAME = L['PLAYLISTS']
PLAYLISTS_PATH = DIR_PATH + PLAYLIST_FOLDER_NAME + os.sep

DL_FOLDER_NAME = L['DOWNLOADS']
DOWNLOAD_PATH = DIR_PATH + DL_FOLDER_NAME + os.sep

# modifying PATH
os.environ['PATH'] = DIR_PATH + 'subprograms' + \
    os.sep + 'libmpv' + os.pathsep + os.environ['PATH']
os.environ['PATH'] = DIR_PATH + 'subprograms' + os.sep + \
    'ffmpeg' + os.sep + 'bin' + os.pathsep + os.environ['PATH']
os.environ['PATH'] = DIR_PATH + 'subprograms' + os.sep + \
    'ffmpeg' + os.pathsep + os.environ['PATH']
os.environ['PATH'] = DIR_PATH + 'subprograms' + \
    os.pathsep + os.environ['PATH']

# file paths
DEFAULT_ART = 'default_art.jpg'  # TODO: remove
DATABASE = DIR_PATH + "databases" + os.sep + "data.sqlite"

# special
FIRST_USE = not path_exists(DATABASE)

# gui
THEME_NAME = 'serenity'
COLOUR_DICT = USER_THEMES['serenity']['colors']
DEFAULT_FONT_FAMILY = 'Cascadia Code Light'
SMALL_SCREEN_CUTOFF = 2000

def preconfigure_style():
    from ttkbootstrap.style import ThemeDefinition
    style = ttk.Style()
    for name, definition in USER_THEMES.items():
        style.register_theme(
            ThemeDefinition(
                name=name,
                themetype=definition["type"],
                colors=definition["colors"],
            )
        )

def configure_style():
    style = ttk.Style(THEME_NAME)
    # the following line stops annoying highlight lines on button click
    style.configure('TButton', focuscolor=style.configure(
        'TButton')['background'])
    style.configure('TFrame', background='black')

def configure_font():
    DEFAULT_FONT = ttk.font.nametofont("TkDefaultFont")
    DEFAULT_FONT_SIZE = 14
    DEFAULT_FONT.configure(family=DEFAULT_FONT_FAMILY, size=DEFAULT_FONT_SIZE)

def configure_root(root: ttk.Window, expanded: bool = False):
    import serenityapp.mastertools as mastertools
    root.attributes('-alpha', 0)
    root.iconbitmap(DIR_PATH + 'icon.ico')
    root.title("serenity")
    root.configure(background=COLOUR_DICT['bg'])
    root.columnconfigure(0, weight=1)
    mastertools.init(root, expanded)


'''

Exponential Search

'''

def binarySearch(arr, left, right, element):
    if right >= left:
        mid = left + (right - left) // 2

        if arr[mid] == element:
            return mid

        elif arr[mid] > element:
            return binarySearch(arr, left, mid - 1, element)

        else:
            return binarySearch(arr, mid + 1, right, elment)

    return -1

def exponentialSearch(arr, element):
    if arr[0] == element:
        return 0

    i = 1
    arrLength = len(arr)
    while i < arrLength and arr[i] <= element:
        i *= 2

    return binarySearch(arr, i / 2, min(i, arrLength), element)

arr = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
element = 4

found = exponentialSearch(arr, element)
if found == -1:
    print('Error | Not found')
else:
    print('Found | At position', found + 1)


import pathlib
import socket
import sys
import time
from datetime import datetime
import secrets
import traceback

from cryptography import x509
from cryptography.exceptions import InvalidSignature
from cryptography.fernet import Fernet
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import padding
from cryptography.hazmat.backends import default_backend

def convert_int_to_bytes(x):
    """
    Convenience function to convert Python integers to a length-8 byte representation
    """
    return x.to_bytes(8, "big")

def convert_bytes_to_int(xbytes):
    """
    Convenience function to convert byte value to integer value
    """
    return int.from_bytes(xbytes, "big")

def main(args):
    port = int(args[0]) if len(args) > 0 else 4321
    server_address = args[1] if len(args) > 1 else "localhost"

    start_time = time.time()

    #Extracting Public Keys from the Certificate:
    f = open("auth/cacsertificate.crt", "rb")
    ca_cert_raw = f.read()
    ca_cert = x509.load_pem_x509_certificate( data=ca_cert_raw, backend=default_backend())
    ca_public_key = ca_cert.public_key()

    # try:
    print("Establishing connection to server...")
    # Connect to server

    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        s.connect((server_address, port))
        print("Connected")

        #Send the authentication message
        s.sendall(convert_int_to_bytes(3))
        nonce = datetime.now(tz=None)
        nonce_bytes = bytes(str(nonce), encoding="utf8")
        s.sendall(convert_int_to_bytes(len(nonce_bytes)))
        s.sendall(nonce_bytes)

        #Receive authentication message:
        signed_message_len = s.recv(8);
        signed_message = s.recv(convert_bytes_to_int(signed_message_len ));

        #Receive signed certificate from server
        server_cert_len = s.recv(8);
        server_cert_raw= s.recv(convert_bytes_to_int( server_cert_len ));

        #Verify Certificate
        server_cert = x509.load_pem_x509_certificate(data=server_cert_raw, backend=default_backend())
        ca_public_key.verify(signature=server_cert.signature, # signature bytes to  verify
         data=server_cert.tbs_certificate_bytes, # certificate data bytes that was signed by CA
         padding=padding.PKCS1v15(), # padding used by CA bot to sign the the server's csr
         algorithm=server_cert.signature_hash_algorithm,
         )

        server_public_key = server_cert.public_key()

        server_public_key.verify(
            signed_message,
            nonce_bytes,
            padding.PSS(
                mgf=padding.MGF1(hashes.SHA256()),
                salt_length=padding.PSS.MAX_LENGTH,
                 ),
                  hashes.SHA256(),
                  )

        if (server_cert.not_valid_before <= datetime.utcnow() <= server_cert.not_valid_after):

            while True:
                filename = input("Enter a filename to send (enter -1 to exit):")

                while filename != "-1" and (not pathlib.Path(filename).is_file()):
                    filename = input("Invalid filename. Please try again:")

                if filename == "-1":
                    s.sendall(convert_int_to_bytes(2))
                    break

                filename_bytes = bytes(filename, encoding="utf8")

                # Send the filename
                s.sendall(convert_int_to_bytes(0))
                s.sendall(convert_int_to_bytes(len(filename_bytes)))
                s.sendall(filename_bytes)

                # Send the file
                with open(filename, mode="rb") as fp:
                    data = fp.read()
                    s.sendall(convert_int_to_bytes(1))
                    s.sendall(convert_int_to_bytes(len(data)))
                    s.sendall(data)

        # Close the connection
        s.sendall(convert_int_to_bytes(2))
        print("Closing connection...")

    end_time = time.time()
    print(f"Program took {end_time - start_time}s to run.")

if __name__ == "__main__":
    main(sys.argv[1:])


#
# This sets up how models are displayed
# in the web admin interface.
#

from django import forms
from django.conf import settings
from django.contrib import admin
from src.typeclasses.admin import AttributeInline, TagInline
from src.objects.models import ObjectDB

class ObjectAttributeInline(AttributeInline):
    model = ObjectDB.db_attributes.through

class ObjectTagInline(TagInline):
    model = ObjectDB.db_tags.through

class ObjectCreateForm(forms.ModelForm):
    "This form details the look of the fields"
    class Meta:
        model = ObjectDB
        fields = '__all__'
    db_key = forms.CharField(label="Name/Key",
                             widget=forms.TextInput(attrs={'size': '78'}),
                             help_text="Main identifier, like 'apple', 'strong guy', 'Elizabeth' etc. If creating a Character, check so the name is unique among characters!",)
    db_typeclass_path = forms.CharField(label="Typeclass",
                                        initial=settings.BASE_OBJECT_TYPECLASS,
                                        widget=forms.TextInput(attrs={'size': '78'}),
                                        help_text="This defines what 'type' of entity this is. This variable holds a Python path to a module with a valid Evennia Typeclass. If you are creating a Character you should use the typeclass defined by settings.BASE_CHARACTER_TYPECLASS or one derived from that.")
    db_cmdset_storage = forms.CharField(label="CmdSet",
                                        initial="",
                                        required=False,
                                        widget=forms.TextInput(attrs={'size': '78'}),
                                        help_text="Most non-character objects don't need a cmdset and can leave this field blank.")
    raw_id_fields = ('db_destination', 'db_location', 'db_home')

class ObjectEditForm(ObjectCreateForm):
    "Form used for editing. Extends the create one with more fields"

    class Meta:
        fields = '__all__'
    db_lock_storage = forms.CharField(label="Locks",
                                      required=False,
                                      widget=forms.Textarea(attrs={'cols':'100', 'rows':'2'}),
                                      help_text="In-game lock definition string. If not given, defaults will be used. This string should be on the form <i>type:lockfunction(args);type2:lockfunction2(args);...")

class ObjectDBAdmin(admin.ModelAdmin):

    inlines = [ObjectTagInline, ObjectAttributeInline]
    list_display = ('id', 'db_key', 'db_player', 'db_typeclass_path')
    list_display_links = ('id', 'db_key')
    ordering = ['db_player', 'db_typeclass_path', 'id']
    search_fields = ['^db_key', 'db_typeclass_path']
    raw_id_fields = ('db_destination', 'db_location', 'db_home')

    save_as = True
    save_on_top = True
    list_select_related = True
    list_filter = ('db_typeclass_path',)
    #list_filter = ('db_permissions', 'db_typeclass_path')

    # editing fields setup

    form = ObjectEditForm
    fieldsets = (
        (None, {
                'fields': (('db_key','db_typeclass_path'), ('db_lock_storage', ),
                           ('db_location', 'db_home'), 'db_destination','db_cmdset_storage'
                           )}),
        )
    #fieldsets = (
    #    (None, {
    #            'fields': (('db_key','db_typeclass_path'), ('db_permissions', 'db_lock_storage'),
    #                       ('db_location', 'db_home'), 'db_destination','db_cmdset_storage'
    #                       )}),
    #    )

    #deactivated temporarily, they cause empty objects to be created in admin

    # Custom modification to give two different forms wether adding or not.
    add_form = ObjectCreateForm
    add_fieldsets = (
        (None, {
                'fields': (('db_key','db_typeclass_path'),
                           ('db_location', 'db_home'), 'db_destination', 'db_cmdset_storage'
                           )}),
        )

    #add_fieldsets = (
    #    (None, {
    #            'fields': (('db_key','db_typeclass_path'), 'db_permissions',
    #                       ('db_location', 'db_home'), 'db_destination', 'db_cmdset_storage'
    #                       )}),
    #    )
    def get_fieldsets(self, request, obj=None):
        if not obj:
            return self.add_fieldsets
        return super(ObjectDBAdmin, self).get_fieldsets(request, obj)

    def get_form(self, request, obj=None, **kwargs):
        """
        Use special form during creation
        """
        defaults = {}
        if obj is None:
            defaults.update({
                    'form': self.add_form,
                    'fields': admin.util.flatten_fieldsets(self.add_fieldsets),
                    })
            defaults.update(kwargs)
        return super(ObjectDBAdmin, self).get_form(request, obj, **defaults)

    def save_model(self, request, obj, form, change):
        obj.save()
        if not change:
            # adding a new object
            obj = obj.typeclass
            obj.basetype_setup()
            obj.basetype_posthook_setup()
            obj.at_object_creation()
        obj.at_init()

admin.site.register(ObjectDB, ObjectDBAdmin)


from turtle import screensize, title
from PyQt5 import QtWidgets
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
import os
import numpy as np
import matplotlib.pyplot as plt
import sys, UI
import cv2
import re
import tkinter as tk

def sorted_alphanumeric(data):
    convert = lambda text: int(text) if text.isdigit() else text.lower()
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
    return sorted(data, key=alphanum_key)

criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.01)

objp = np.zeros((8*11,3), np.float32)
objp[:,:2] = np.mgrid[0:11,0:8].T.reshape(-1,2)
objpoints = []
imgpoints = []
InputW=""
imgLL=[]
imgRL=[]
images=[]
filenames=[]
alert1="****PRESS Q TO EXIT DRWING****"
alert="*****PLEASE DON'T CLOSE THIS WINDOW!!*****"
def cacu():
    objpoints.clear()
    imgpoints.clear()
    for img in images:
        gray=cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
        ret, corner=cv2.findChessboardCorners(gray, (11,8),None)
        if ret == True:
                objpoints.append(objp)
                corners2 = cv2.cornerSubPix(gray,corner, (11,11), (-1,-1), criteria)
                imgpoints.append(corners2)

def load_images_from_folder(folder):
    images.clear()
    filenames.clear()
    for filename in sorted_alphanumeric(os.listdir(folder)):
        img = cv2.imread(os.path.join(folder,filename))
        if img is not None:
            filenames.append(filename)
            images.append(img)

def draw1(image,imgpts):
    image = cv2.line(image, tuple(imgpts[0].ravel()), tuple(imgpts[1].ravel()), (0, 0, 255), 5)
    return image

class myMainWindow(QMainWindow, UI.Ui_MainWindow):
    def __init__(self):
         super().__init__()
         self.setupUi(self)
         self.pushButton.clicked.connect(self.Load_Image)
         self.pushButton_11.clicked.connect(self.Find_corner)
         self.pushButton_10.clicked.connect(self.Find_IN)
         self.pushButton_12.clicked.connect(self.Find_EI)
         self.pushButton_13.clicked.connect(self.Find_DI)
         self.pushButton_14.clicked.connect(self.Undis)
         self.pushButton_15.clicked.connect(self.SWB)
         self.pushButton_18.clicked.connect(self.SWV)
         self.pushButton_2.clicked.connect(self.LF)
         self.pushButton_3.clicked.connect(self.LR)
         self.pushButton_19.clicked.connect(self.STM)
    def STM(self):
        imgL=imgLL[0]
        imgR=imgRL[0]
        tempL=cv2.cvtColor(imgL,cv2.COLOR_BGR2GRAY)
        tempR=cv2.cvtColor(imgR,cv2.COLOR_BGR2GRAY)
        stereo = cv2.StereoBM_create(256,25)
        disparity = stereo.compute(tempL, tempR).astype(np.float32)/16.0
        m = np.ones(disparity.shape, dtype="float32")
        disparity = cv2.add(disparity, m)
        disparity=disparity.astype(np.uint8)
        disparityshow= cv2.resize(disparity , (700, 425), interpolation=cv2.INTER_AREA)
        cv2.namedWindow('disparity '+alert)
        cv2.resizeWindow('disparity '+alert,700,425)
        cv2.imshow('disparity '+alert, disparityshow)
        imgL=cv2.resize(imgLL[0] , (700, 425), interpolation=cv2.INTER_AREA)
        imgR=cv2.resize(imgRL[0] , (700, 425), interpolation=cv2.INTER_AREA)
        tempL=cv2.cvtColor(imgL,cv2.COLOR_BGR2GRAY)
        tempR=cv2.cvtColor(imgR,cv2.COLOR_BGR2GRAY)
        stereo = cv2.StereoBM_create(128,13)
        disparity = stereo.compute(tempL, tempR)/16
        global tempRR
        tempRR=imgR.copy()
        global tempLL
        tempLL=imgL.copy()
        def draw_circle(event, x, y, flags, param):
            if event == cv2.EVENT_LBUTTONDOWN:
                global tempRR
                tempRR=imgR.copy()
                global tempLL
                tempLL=imgL.copy()
                cv2.circle(tempLL,(x,y),2,(0,255,0),2)
                cv2.circle(tempRR,(int(x-disparity[y][x]),y),2,(0,255,0),2)
        while(1):
            cv2.namedWindow('imageL '+alert1)
            cv2.resizeWindow('imageL '+alert1,700, 425)
            cv2.imshow('imageL '+alert1, tempLL)
            cv2.namedWindow('imageR '+alert1)
            cv2.resizeWindow('imageR '+alert1,700, 425)
            cv2.imshow('imageR '+alert1, tempRR)
            cv2.setMouseCallback('imageL '+alert1, draw_circle)
            if cv2.waitKey(20) & 0xFF ==ord('q'):
                break
        cv2.destroyAllWindows()

    def LF(self):
        imgLL.clear()
        filename,_=QFileDialog().getOpenFileName(self,"Load Image_L")
        if(filename==''): return
        temp= cv2.imread(filename)
        imgLL.append(temp)
    def LR(self):
        imgRL.clear()
        filename,_=QFileDialog().getOpenFileName(self,"Load Image_R")
        if(filename==''): return
        temp= cv2.imread(filename)
        imgRL.append(temp)
    def SWB(self):
        if self.lineEdit.text() != '':
            InputW=self.lineEdit.text()
        i=-1
        for img in images:
            temp=img.copy()
            i=i+1
            gray=cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
            cacu()
            ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
            for x in range(len(InputW)) :
                fs = cv2.FileStorage('alphabet_lib_onboard.txt', cv2.FILE_STORAGE_READ)
                ch = fs.getNode(InputW[x]).mat()
                if ((x+1)%3)==0 :
                    movex=1
                else:
                    movex=7-(((x+1)%3)-1)*3
                if ((x+1)<4):
                    movey=5
                else:
                    movey=2
                for l in range(len(ch)):
                    ch[l][0][0]=ch[l][0][0]+movex
                    ch[l][0][1]=ch[l][0][1]+movey
                    ch[l][1][0]=ch[l][1][0]+movex
                    ch[l][1][1]=ch[l][1][1]+movey
                    axis = np.float32(ch[l]).reshape(-1, 3)
                    imgpoints2, _ =cv2.projectPoints(axis,rvecs[i],tvecs[i],mtx,dist)
                    temp = draw1(temp,imgpoints2)
            cv2.namedWindow('Show word '+InputW+' on '+str(filenames[i]),0)
            cv2.resizeWindow('Show word '+InputW+' on '+str(filenames[i]),1080,900)
            cv2.imshow('Show word '+InputW+' on '+str(filenames[i]),temp)
            cv2.waitKey(1000)
            cv2.destroyAllWindows()

    def SWV(self):
        if self.lineEdit.text() != '':
            InputW=self.lineEdit.text()
        i=-1
        for img in images:
            i=i+1
            temp=img.copy()
            gray=cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
            cacu()
            ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
            for x in range(len(InputW)) :
                fs = cv2.FileStorage('alphabet_lib_vertical.txt', cv2.FILE_STORAGE_READ)
                ch = fs.getNode(InputW[x]).mat()
                if ((x+1)%3)==0 :
                    movex=1
                else:
                    movex=7-(((x+1)%3)-1)*3
                if ((x+1)<4):
                    movey=5
                else:
                    movey=2
                for l in range(len(ch)):
                    ch[l][0][0]=ch[l][0][0]+movex
                    ch[l][0][1]=ch[l][0][1]+movey
                    ch[l][1][0]=ch[l][1][0]+movex
                    ch[l][1][1]=ch[l][1][1]+movey
                    axis = np.float32(ch[l])
                    imgpoints2, _ =cv2.projectPoints(axis,rvecs[i],tvecs[i],mtx,dist)
                    temp = draw1(temp,imgpoints2)
            cv2.namedWindow('Show word '+InputW+' on '+str(filenames[i]),0)
            cv2.resizeWindow('Show word '+InputW+' on '+str(filenames[i]),1080,900)
            cv2.imshow('Show word '+InputW+' on '+str(filenames[i]),temp)
            cv2.waitKey(1000)
            cv2.destroyAllWindows()

    def Undis(self):
        j=-1
        for img in images:
            j=j+1
            if(j==15): break
            gray=cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
            ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
            size = gray.shape
            w=size[1]
            h=size[0]
            newcameramtx, roi = cv2.getOptimalNewCameraMatrix(mtx, dist, (w,h), 1, (w,h))
            dst = cv2.undistort(img, mtx, dist, None, newcameramtx)
            dst=cv2.cvtColor(dst,cv2.COLOR_RGB2GRAY)
            imgs=np.hstack((gray,dst))
            cv2.namedWindow('undistor '+str(filenames[j]),0)
            cv2.resizeWindow('undistor '+str(filenames[j]),1400,900)
            cv2.imshow('undistor '+str(filenames[j]),imgs)
            cv2.waitKey(750)
            cv2.destroyAllWindows()

    def Find_DI(self):
        img=images[0]
        gray=cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
        ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
        print("1.4 Find the Distortion Matrix :")
        print(dist)
        print(" ")

    def Find_EI(self):
        value=int(self.comboBox.currentText())
        img=images[0]
        gray=cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
        ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
        rvecs,_=cv2.Rodrigues(rvecs[value-1])
        new_arr=np.concatenate([rvecs,tvecs[value-1]],1)
        print("1.3 Find the Extrinsic Matrix with picuture of " +str(value)+": ")
        print(new_arr)
        print(" ")

    def Find_IN(self):
        img=images[0]
        gray=cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
        ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
        print("1.2 Find the Intrinsic Matrix : ")
        print(mtx)
        print(" ")
    def Find_corner(self):
        objpoints.clear()
        imgpoints.clear()
        i=-1
        for img in images:
            i=i+1
            gray=cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
            ret, corner=cv2.findChessboardCorners(gray, (11,8),None)
            if ret == True:
                objpoints.append(objp)
                corners2 = cv2.cornerSubPix(gray,corner, (11,11), (-1,-1), criteria)
                imgpoints.append(corner)
                cv2.namedWindow('corner '+str(filenames[i]),0)
                cv2.resizeWindow('corner '+str(filenames[i]),700,900)
                temp=img.copy()
                cv2.drawChessboardCorners(temp, (11, 8), corners2, ret)
                cv2.imshow('corner '+str(filenames[i]),temp)
                cv2.waitKey(500)
                cv2.destroyAllWindows()

    def Load_Image(self):
        filename=QFileDialog().getExistingDirectory(self,"Load Image")
        if(filename==''): return
        load_images_from_folder(filename)

if __name__ == '__main__':
    app = QtWidgets.QApplication(sys.argv)
    window = myMainWindow()
    window.show()
    sys.exit(app.exec_())


#  https://docs.python.org/2/tutorial/errors.html
import sys

if __name__ == "__main__":
    try:
        f = open('test.txt')
        s = f.readline()
        i = int(s.strip())
    except Exception as e:
        print('Reason:', e)


from urllib.request import urlopen
from bs4 import BeautifulSoup as BS

response = urlopen("https://www.flipkart.com/search?q=watches&as=on&as-show=on&otracker=AS_Query_TrendingAutoSuggest_2_0_na_na_na&otracker1=AS_Query_TrendingAutoSuggest_2_0_na_na_na&as-pos=2&as-type=HISTORY&suggestionId=watches&requestId=403378b9-ed67-466b-b64a-bb79995da765")

html = BS(response, "lxml")

products_list = html.find_all("div", "IIdQZO _1SSAGr")
# print(len(products_list))
for i in range(len(products_list)):
    print(products_list[i])
    brand_name = products_list[i].find("div", "_2B_pmu").text
    product_name = products_list[i].find("a", "_2mylT6").text
    current_price = products_list[i].find("div", "_1vC4OE").text
    mrp = products_list[i].find("div", "_3auQ3N")
    mrp = mrp.text if mrp != None else current_price
    discount = products_list[i].find("div", "VGWI6T")

    # if discount != None:
    #     discount = discount.text
    # else:
    #     discount = "Discount not available"

    discount = discount.text if discount != None else "Discount not available"

    # variable_name = condition ? true_action : false_action //ternary operator

    print(f'''
    Product No. {i + 1}
    Product name - {brand_name} {product_name}
    Price - {current_price}
    MRP - {mrp}
    Discount - {discount}
    ''')


import os
from typing import Sequence

import gym
import torch
from torch import nn

from allenact.base_abstractions.sensor import SensorSuite, Sensor
from allenact.embodiedai.mapping.mapping_models.active_neural_slam import (
    ActiveNeuralSLAM,
)
from allenact.utils.misc_utils import multiprocessing_safe_download_file_from_url
from allenact_plugins.ithor_plugin.ithor_sensors import (
    RelativePositionChangeTHORSensor,
    ReachableBoundsTHORSensor,
)
from baseline_configs.one_phase.one_phase_rgb_il_base import (
    OnePhaseRGBILBaseExperimentConfig,
)
from rearrange.baseline_models import OnePhaseRearrangeActorCriticFrozenMap
from rearrange.constants import (
    PICKUPABLE_OBJECTS,
    OPENABLE_OBJECTS,
)
from rearrange_constants import ABS_PATH_OF_REARRANGE_TOP_LEVEL_DIR

class OnePhaseRGBResNetFrozenMapDaggerExperimentConfig(
    OnePhaseRGBILBaseExperimentConfig
):
    CNN_PREPROCESSOR_TYPE_AND_PRETRAINING = (
        None  # Not necessary as we're handling things in the model
    )
    IL_PIPELINE_TYPE = "40proc"

    ORDERED_OBJECT_TYPES = list(sorted(PICKUPABLE_OBJECTS + OPENABLE_OBJECTS))

    MAP_RANGE_SENSOR = ReachableBoundsTHORSensor(margin=1.0)

    MAP_INFO = dict(
        map_range_sensor=MAP_RANGE_SENSOR,
        vision_range_in_cm=40 * 5,
        map_size_in_cm=1050
        if isinstance(MAP_RANGE_SENSOR, ReachableBoundsTHORSensor)
        else 2200,
        resolution_in_cm=5,
    )

    @classmethod
    def sensors(cls) -> Sequence[Sensor]:
        return list(
            super(OnePhaseRGBResNetFrozenMapDaggerExperimentConfig, cls).sensors()
        ) + [RelativePositionChangeTHORSensor(), cls.MAP_RANGE_SENSOR,]

    @classmethod
    def tag(cls) -> str:
        return f"OnePhaseRGBResNetFrozenMapDagger_{cls.IL_PIPELINE_TYPE}"

    @classmethod
    def create_model(cls, **kwargs) -> nn.Module:
        map_kwargs = dict(
            frame_height=224,
            frame_width=224,
            vision_range_in_cm=cls.MAP_INFO["vision_range_in_cm"],
            resolution_in_cm=cls.MAP_INFO["resolution_in_cm"],
            map_size_in_cm=cls.MAP_INFO["map_size_in_cm"],
        )

        observation_space = (
            SensorSuite(cls.sensors()).observation_spaces
            if kwargs.get("sensor_preprocessor_graph") is None
            else kwargs["sensor_preprocessor_graph"].observation_spaces
        )

        semantic_map_channels = len(cls.ORDERED_OBJECT_TYPES)
        height_map_channels = 3
        map_kwargs["n_map_channels"] = height_map_channels + semantic_map_channels
        frozen_map = ActiveNeuralSLAM(**map_kwargs, use_resnet_layernorm=True)

        pretrained_map_ckpt_path = os.path.join(
            ABS_PATH_OF_REARRANGE_TOP_LEVEL_DIR,
            "pretrained_model_ckpts",
            "pretrained_active_neural_slam_via_walkthrough_75m.pt",
        )
        multiprocessing_safe_download_file_from_url(
            url="https://prior-model-weights.s3.us-east-2.amazonaws.com/embodied-ai/rearrangement/walkthrough/pretrained_active_neural_slam_via_walkthrough_75m.pt",
            save_path=pretrained_map_ckpt_path,
        )
        frozen_map.load_state_dict(
            torch.load(pretrained_map_ckpt_path, map_location="cpu",)
        )

        return OnePhaseRearrangeActorCriticFrozenMap(
            map=frozen_map,
            action_space=gym.spaces.Discrete(len(cls.actions())),
            observation_space=observation_space,
            rgb_uuid=cls.EGOCENTRIC_RGB_UUID,
            unshuffled_rgb_uuid=cls.UNSHUFFLED_RGB_UUID,
            semantic_map_channels=semantic_map_channels,
            height_map_channels=height_map_channels,
        )


import pandas as pd
import numpy as np
import os
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report

keyword_list = ['obese',  'obesity','asthma', 'atherosclerotic', 'cardiovascular','CAD', 'congestive', 'CHF', 'depression', 'diabetes''mellitus', 'DM', 'gallstones', 'cholecystectomy', 'gastroesophageal', 'reflux', 'GERD', 'gout', 'hypercholesterolemia', 'hypertension', 'HTN', 'hypertriglyceridemia', 'obstructive', 'apnea', 'OSA', 'osteoarthritis','OA', 'peripheral','vascular', 'PVD', 'venous','mg', 'MG']
section_categories = ['Diagnosis', 'Past or Present History of Illness', 'Social/Family History', 'Physical or Laboratory Examination', 'Medication.Disposition', 'Other']
train_dir = './Data/Train_Textual'
valid_dir = './Data/Test_Intuitive'
test_dir = './Data/Validation'

def get_dataset(train_dir, valid_dir):
    train_df = pd.DataFrame(columns=('text', 'label'))
    valid_df = pd.DataFrame(columns=('text', 'label'))

    cnt = 0
    for path in os.listdir(train_dir):
        with open(os.path.join(train_dir, path), 'r') as f:
            if path.startswith('U'):
                train_df.loc[cnt] = [f.read(), 0]
            else:
                train_df.loc[cnt] = [f.read(), 1]
            cnt += 1
    cnt = 0
    for path in os.listdir(valid_dir):
        with open(os.path.join(valid_dir, path), 'r') as f:
            if path.startswith('N'):
                valid_df.loc[cnt] = [f.read(), 0]
            else:
                valid_df.loc[cnt] = [f.read(), 1]
            cnt += 1

    return train_df, valid_df

def predict_testset_rule_based():
    test_data_df = pd.DataFrame(columns=('Filename', 'text'))
    output_csv = pd.DataFrame(columns=('Filename', 'Obesity'))

    cnt = 0
    for path in os.listdir(test_dir):
        with open(os.path.join(test_dir, path), 'r') as f:
            test_data_df.loc[cnt] = [path, f.read()]
            cnt += 1

    pred = []
    cnt = 0
    for i in range(len(test_data_df)):
        for j in range(len(keyword_list)):
            if keyword_list[j] in test_data_df.iloc[i,1]:
                pred.append(1)
                cnt+=1
                break
            elif j == len(keyword_list) - 1:
                pred.append(0)
                cnt+=1

    output_csv['Filename'] = test_data_df['Filename']
    output_csv['Obesity'] = pred
    output_csv = output_csv.sort_values(by=['Filename'])
    output_csv.to_csv('./output.csv', index=0)

def getKeywordCount(df):
    corpus = ''
    for i in range(len(df.iloc[:,0])):
        corpus+=df.iloc[i,0]

    counts = dict()
    words = corpus.split()

    for word in words:
        if word in counts:
            counts[word] += 1
        else:
            counts[word] = 1
    sorted(counts.items(), key=lambda x:x[1])

    for keyword in keyword_list:
        print(keyword, ': ', counts.get(keyword))

if __name__ == '__main__':
    train_df, valid_df = get_dataset(train_dir, valid_dir)

    getKeywordCount(train_df)
    getKeywordCount(valid_df)

    # df = pd.concat([train_df, valid_df])
    # x_train, x_valid, y_train, y_valid = train_test_split(df['text'], df['label'], test_size=0.33)

    pred = []
    cnt = 0
    for i in range(len(train_df)):
        for j in range(len(keyword_list)):
            if keyword_list[j] in train_df.iloc[i,0]:
                pred.append(1)
                cnt+=1
                break
            elif j == len(keyword_list) - 1:
                pred.append(0)
                cnt+=1
    print(f"train accuracy score: {accuracy_score(train_df.iloc[:,1].tolist(), pred)}")
    print(f"train report:\n{classification_report(train_df.iloc[:,1].tolist(), pred)}")

    pred = []
    cnt = 0
    for i in range(len(valid_df)):
        for j in range(len(keyword_list)):
            if keyword_list[j] in valid_df.iloc[i,0]:
                pred.append(1)
                cnt+=1
                break
            elif j == len(keyword_list) - 1:
                pred.append(0)
                cnt+=1
    print(f"valid accuracy score: {accuracy_score(valid_df.iloc[:,1].tolist(), pred)}")
    print(f"valid report:\n{classification_report(valid_df.iloc[:,1].tolist(), pred)}")

    predict_testset_rule_based()


_PATH = '../../../datasets/'
import os, sys
if _PATH not in sys.path:
    sys.path.insert(0, _PATH)

import tensorflow as tf
import tensorflow_datasets as tfds
from trainer.datasets.utils import normalize, random_jitter
import numpy as np
from skimage.transform import radon,iradon
import glob

def deeplesion(config_name='abnormal',
               split='train',
               size=(512, 512, 1),
               augment=True,
               data_dir=None):
  """Returns a `tf.data.Dataset` generator, in which each element is an (-1024, 200) HU windowed and 0-1 normalized image

  'abnormal':
    1. take out 'image'
    2. convert pixel intensities to the original Hounsfield unit (HU) values
    3. clip pixel values into the range (-1024, 200)
    4. random jitter the image if augment is True else resize the image to 'size'
    5. normalize pixel values to (0,1)

  Args:
    config_name: `str`, name of the configure
    split: `str`, ['train' | 'val' | 'test']
    size: `tuple`, (height, width, channels)
    batch_size: `int`
    augment: `Bool`
    data_dir: `str`, optional, where to store the downloaded and serialized data

  Returns:
    `tuple`, (tf.data.Dataset, number of examples)
  """
  count = {'train': 42600, 'validation': 9044, 'test': 9278}

  data = tfds.load('deeplesion/{}'.format(config_name), data_dir=data_dir)
  dataset = data[split]

  dataset = dataset.map(lambda x: (x['image']),
                        num_parallel_calls=tf.data.experimental.AUTOTUNE)
  dataset = dataset.map(lambda x: (_debias_HU(x)),
                        num_parallel_calls=tf.data.experimental.AUTOTUNE)
  dataset = dataset.map(lambda x: (tf.clip_by_value(x, -1024, 200)),
                        num_parallel_calls=tf.data.experimental.AUTOTUNE)
  # augment
  if augment:
      dataset = dataset.map(lambda x:
                            (random_jitter(x, [size[0], size[1], 1])),
                            num_parallel_calls=tf.data.experimental.AUTOTUNE)
  else:
      dataset = dataset.map(lambda x: (tf.image.resize(
          x, [size[0], size[1]],
          method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)),
                            num_parallel_calls=tf.data.experimental.AUTOTUNE)

  dataset = dataset.map(lambda x: (normalize(x)),
                        num_parallel_calls=tf.data.experimental.AUTOTUNE)

  return dataset, count[split]

def deeplesion_lr_hr_pair(split='train', size=(512, 512, 1), batch_size=8, factor=4, augment=False, data_dir=None):
  """Returns a `tf.data.Dataset` generator, in which each element is an image pair of its low resolution and high resolution representatives

  generate radon space down sampled data and save the data into disk if the data has not existed.
  load the data and batch

  Args:
    split: `str`, ['train' | 'val' | 'test']
    size: `tuple`, (height, width, channels)
    batch_size: `int`
    factor: `in`, downsampling factor
    augment: `Bool`
    data_dir: `str`, optional, where to store the downloaded and serialized data

  Returns:
    `tuple`, (tf.data.Dataset, number of examples)
  """
  count = {'train': 2556,
           'validation': 278,
          }
  PATH_PREFIX = os.path.join(data_dir, "custom", 'sisr_deeplesion', split)
  records = sorted(glob.glob(os.path.join(PATH_PREFIX, '*.tfrecord')))

  if not records:
    # preprocess the data and save to disk
    dataset, cnt = deeplesion('abnormal', split=split, size=size, augment=augment, data_dir=data_dir)

    # build pair of lr and hr
    dataset = dataset.map(lambda x: get_pair_wrapper(x, factor, size),
                          num_parallel_calls=tf.data.experimental.AUTOTUNE)
    dataset = dataset.map(lambda x, y: (normalize(x), normalize(y)),
                          num_parallel_calls=tf.data.experimental.AUTOTUNE)

    # lr_hr_pair uses only a subset of total dataset
    assert count[split] <= cnt

    # write to tfrecord because the radon function takes too much time
    NUM_SHARDS = int(count[split] // 10)
    dataset_to_write = dataset.map(lambda x, y: tf.concat([x, y], -1),
                          num_parallel_calls=tf.data.experimental.AUTOTUNE)
    dataset_to_write = dataset_to_write.take(count[split])
    dataset_to_write = dataset_to_write.map(tf.io.serialize_tensor)
    dataset_to_write = dataset_to_write.enumerate()

    def reduce_func(key, dataset):
      filename = tf.strings.as_string(key, width=3, fill='0') + '.tfrecord'
      filename = tf.strings.join([PATH_PREFIX, '/', filename])
      writer = tf.data.experimental.TFRecordWriter(filename)
      writer.write(dataset.map(lambda _, x: x))
      return tf.data.Dataset.from_tensors(filename)

    dataset_to_write = dataset_to_write.apply(tf.data.experimental.group_by_window(
                          lambda i, _: i % NUM_SHARDS, reduce_func, tf.int64.max))

    for x in dataset_to_write:
      print(x)

  # load data from disk
  dataset = tf.data.TFRecordDataset(records)
  dataset = dataset.map(lambda x: tf.io.parse_tensor(x, tf.float32))
  dataset = dataset.map(setup_shape_after_load,
                       num_parallel_calls=tf.data.experimental.AUTOTUNE)
  dataset = dataset.map(lambda x: tf.unstack(x, num=2, axis=-1),
                       num_parallel_calls=tf.data.experimental.AUTOTUNE)
  dataset = dataset.map(lambda x, y: (tf.expand_dims(x, -1), tf.expand_dims(y, -1)),
                       num_parallel_calls=tf.data.experimental.AUTOTUNE)
  # batch
  dataset = dataset.repeat().batch(batch_size).prefetch(16)

  return dataset, count[split]

def setup_shape_after_load(ele):
  """set tensor shape and return"""
  ele.set_shape((512, 512, 2))
  return ele

def get_pair_wrapper(hr, downsampling_factor, size):
  """wrapper of py_function `get_LR_HR_pair`"""
  pair = tf.py_function(func=get_LR_HR_pair, inp=[hr, downsampling_factor], Tout=[tf.float32, tf.float32])
  pair[0].set_shape(size)
  pair[1].set_shape(size)
  return pair

def get_LR_HR_pair(hr, factor):
  """Returns an image pair

  Downsample image in random space with theta axis

  Attributs:
    hr: `tensor`, ground truth image
    factor: `int`, downsampling factor

  Returns:
    an `numpy.array` image pair in cartesian space
  """
  gt=np.squeeze(hr.numpy())
  theta = np.linspace(0.0, 180.0, 1000, endpoint=False)
  sinogram = radon(gt, theta=theta, circle=False) # A collection of projections at several angles is called a sinogram, which is a linear transform of the original image

  theta_down = theta[0:1000:factor]
  sparse =iradon(sinogram[:,0:1000:factor],theta=theta_down,circle=False) # FBP is used here
  full =iradon(sinogram,theta=theta,circle=False)

  lr = sparse[:,:, None]
  hr = full[:,:, None]
  return lr, hr

def _debias_HU(image):
  """Returns image with HU values

  subtract 32768 from the pixel intensities to obtain the original Hounsfield unit (HU) values

  Args:
    image: `3D or 4D tf.Tensor`, shape=([None], 512, 512, 1)

  Return:
    `tf.Tensor`, shape=(512, 512, 1)
  """
  image = tf.cast(image, dtype=tf.int32)
  image = tf.cast(image - 32768, dtype=tf.int64)
  return image


# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import os
from datetime import datetime

from scrapy.exceptions import DropItem
from scrapy.exporters import JsonLinesItemExporter

class ZefixScraperPipeline(object):

    def __init__(self):
        self.items_exporter = None
        self.processed_uids = set()

    def open_spider(self, spider):
        destination_file = spider.settings.get("DESTINATION_FILE")
        destination_dir = os.path.dirname(destination_file)
        os.makedirs(destination_dir, exist_ok=True)
        if os.path.exists(destination_file):
            os.replace(destination_file, '{}_backup_{}'.format(destination_file, datetime.utcnow().isoformat()))
        file_ = open(destination_file, 'ab+')
        company_exporter = JsonLinesItemExporter(file_)
        company_exporter.start_exporting()
        self.items_exporter = company_exporter

    def close_spider(self, spider):
        self.items_exporter.finish_exporting()

    def _exporter_for_item(self, item):
        return self.items_exporter

    def process_item(self, item, spider):
        # if item["UID"] in self.processed_uids:
        #     raise DropItem("Duplicate item found: %s" % item)
        # self.processed_uids.add(item["UID"])
        exporter = self._exporter_for_item(item)
        exporter.export_item(item)
        return item


def solution(array, commands):
    answer = []
    print(array)
    for arr in commands:
        v = array[(arr[0]-1):arr[1]]
        v.sort()
        answer.append(v[arr[2]-1])
    return answer

print(solution([1, 5, 2, 6, 3, 7, 4],[[2, 5, 3], [4, 4, 1], [1, 7, 3]]))


import pandas as pd
import numpy as np
import math
from pathlib import Path
import arrow
from src.data.gym.servo import ServoSimulation, create_old, create_rd50
import matplotlib.pyplot as plt

_TRAJECTORY_DUMP_PATH = Path(__file__).resolve().parents[3].joinpath('data/raw/')

def trajectory_with_current_to_csv(servo, trajectory):
    time = arrow.utcnow().format('[YYYY-MM-DD][HH:mm:ss]')
    fname = str(_TRAJECTORY_DUMP_PATH) + '/{!s}{!r}.csv'.format(time, servo)
    pd.DataFrame(trajectory, columns=['mpa', 'mva', 'lpa', 'lva', 'ac']).to_csv(fname, index=False)

class ServoTrajectoryGenerator(object):
    def __init__(self, servo: ServoSimulation):
        self.servo = servo
        self.trajectories = []

    def __check_traj_time_more_than_dt(self, time_s):
        if time_s < self.servo.dt:
            raise ValueError("time_s < servo.dt")

    def __calculate_number_of_steps(self, time_s):
        return math.ceil(time_s / self.servo.dt)

    def generate_with_const_current_no_load(self, current, time_s):
        self.__check_traj_time_more_than_dt(time_s)
        number_of_steps = self.__calculate_number_of_steps(time_s)
        states = [self.servo.step_with_applied_current(current) for _ in range(number_of_steps)]
        trajectory = np.array(states)
        self.trajectories.append(trajectory)
        return trajectory

    def generate_with_noisy_uniform_const_current_no_load(self, current, time_s, lower_bound, upper_bound):
        self.__check_traj_time_more_than_dt(time_s)
        number_of_steps = self.__calculate_number_of_steps(time_s)
        noise = np.random.uniform(lower_bound, upper_bound, number_of_steps)
        states = [self.servo.step_with_applied_current(np.clip(current + n, -self.servo.max_current, self.servo.max_current)) for n in noise]
        trajectory = np.array(states)
        self.trajectories.append(trajectory)
        return trajectory

    def generate_with_noisy_gaussian_const_current_no_load(self, current, time_s, mean, std):
        self.__check_traj_time_more_than_dt(time_s)
        number_of_steps = self.__calculate_number_of_steps(time_s)
        noise = np.random.normal(mean, std, number_of_steps)
        states = [self.servo.step_with_applied_current(np.clip(current + n, -self.servo.max_current, self.servo.max_current)) for n in noise]
        trajectory = np.array(states)
        self.trajectories.append(trajectory)
        return trajectory

    def clear_trajectories(self):
        self.trajectories.clear()

class RandomTrajectoryGenerationProcess(object):
    def __init__(self, generator: ServoTrajectoryGenerator, n_pieces, max_piece_time):
        self.generator = generator
        self.servo = self.generator.servo
        self.pieces = n_pieces
        self.max_trajectory_time = max_piece_time

    def run_uniform(self):
        for _ in range(self.pieces):
            time = np.random.uniform(0.05, self.max_trajectory_time)
            current = np.random.uniform(-self.servo.max_current, self.servo.max_current)
            noize = abs(current) * 0.05
            self.generator.generate_with_noisy_uniform_const_current_no_load(current, time, -noize, noize)
        trajectory = np.concatenate(self.generator.trajectories)
        return trajectory

if __name__ == '__main__':
    np.random.seed(42)
    s = create_rd50(dt=0.0000625)
    traj = RandomTrajectoryGenerationProcess(
        ServoTrajectoryGenerator(s),
        10, 0.5
    ).run_uniform()
    trajectory_with_current_to_csv(s, traj)


import copy

file_names = dict()
file_names["a"] = "a_example.txt"
file_names["b"] = "b_lovely_landscapes.txt"
file_names["c"] = "c_memorable_moments.txt"
file_names["d"] = "d_pet_pictures.txt"
file_names["e"] = "e_shiny_selfies.txt"

# get txt file
input_path = "inputs/"
fname = input_path + file_names["b"]
input_file_name = "b_to_input.txt"
output_file_name = "b_output_random.txt"
with open(fname) as f:
    content = f.readlines()
    content = [x.strip() for x in content]

# prase through input file and put content in a list
N = int(content[0])

# storage is a list of tuples of (orientation, [attributes], [id])
storage = [0] * N
for i in range(1, N+1):
    line = content[i].split()
    orientation = line[0]
    attributes = set(line[2:])
    item = (orientation, attributes, [i - 1])
    storage[i-1] = item

# give the score of the output list
# ex: [[1], [0,2]]
def score(output_list):
    score = 0
    M = len(output_list)
    for i in range(M-1):
        S_left = output_list[i]
        S_right = output_list[i+1]

        attributes_left = set()
        attributes_right = set()

        for photo_id in S_left:
            photo_attributes = storage[photo_id][1]
            attributes_left = attributes_left.union(photo_attributes)

        for photo_id in S_right:
            photo_attributes = storage[photo_id][1]
            attributes_right = attributes_right.union(photo_attributes)

        num_intersection = len(attributes_left.intersection(attributes_right))
        num_left = len(attributes_left.difference(attributes_right))
        num_right = len(attributes_right.difference(attributes_left))

        score += min(num_intersection, num_left, num_right)

    return score

# helper function ot write the solution list to a txt file
def write_output(filename, solution):
    complete_file_name = filename + ".txt"
    with open(filename, "w+") as f:
        f.write(str(len(solution)) + "\n")
        for photo_ids in solution:
            line = " ".join([str(num) for num in photo_ids])
            line += "\n"
            f.write(line)

# approach 1, randomly assign vertical pairing and then optimize/finefune
vertical_nodes = []
horizontal_nodes = []
for i in range(len(storage)):
    photo = storage[i]
    if photo[0] == "V":
        vertical_nodes.append(photo)
    else:
        horizontal_nodes.append(photo)

import random
# randomly shuffle the list
random.shuffle(vertical_nodes)

merged_nodes = [0] * (len(vertical_nodes) // 2)
for i in range(len(merged_nodes)):
    left_vertical_photo = vertical_nodes[2*i]
    right_vertical_photo = vertical_nodes[2*i + 1]
    orientation = "V"
    indexes = left_vertical_photo[2] + right_vertical_photo[2]
    tags = left_vertical_photo[1].union(right_vertical_photo[1])
    merged_nodes[i] = [orientation, tags, indexes]

list_for_tsp = merged_nodes + horizontal_nodes
random.shuffle(list_for_tsp)
solution_list = [item[2] for item in list_for_tsp]
best_list = copy.deepcopy(solution_list)
best_score = score(solution_list)
for i in range(0, 1000):
    random.shuffle(list_for_tsp)
    solution_list = [item[2] for item in list_for_tsp]
    score_i = score(solution_list)
    if (score_i > best_score):
        best_list = copy.deepcopy(solution_list)
        best_score = score_i
        print(best_score)
    if i % 50 == 0:
        write_output(output_file_name, best_list)
write_output(output_file_name, best_list)