import discord
import aiohttp
import asyncio
import json
from discord.utils import get
from discord.ext import commands
from discord import guild
from discord.channel import DMChannel

rolesF = json.loads(open("config/roles.json", "r").read())

devLeadID = rolesF["devLead"]
devID = rolesF["dev"]

# Role managment functions.
def hasRole(member, id):
        for role in member.roles:
                if id == role.id:
                        return True
        return False

# Actual COG and command.
class DevRole(commands.Cog,name="BOT Dev"):
        def __init__(self,bot):
                self.bot = bot

        @commands.command(description="Toggles bot-dev role to someone.", usage="", hidden=True)
        async def botdev(self, ctx, member: discord.Member):

                devRole = ctx.guild.get_role(devID)

                # Check if the user has the requiered role to issue the command. (DEV LEAD)
                if (hasRole(ctx.author, devLeadID) and not (hasRole(member, devID))):
                        await member.add_roles(devRole)
                        await ctx.send("Welcome on the BOT Dev team, " + member.mention + "!")
                elif (hasRole(ctx.author, devLeadID) and (hasRole(member, devID))):
                        await member.remove_roles(devRole)
                        await ctx.send(member.mention + " left the BOT Dev team!")
                else:
                        await ctx.send("Sorry, " + ctx.author.mention + " but you do not have the permission to do that.")

def setup(bot):
        bot.add_cog(DevRole(bot))


import requests
import time
import json
import re
import urllib
# Webdriver
from selenium.webdriver import Firefox
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
# Automatic firefox selenium driver
from webdriver_manager.firefox import GeckoDriverManager
# Get environment variables
from dotenv import dotenv_values
# Utility
from utils import fetch_profile_path

url = 'https://www.facebook.com/api/graphql/'
base_referrer = "https://www.facebook.com/groups/{GROUP_ID}/members"
query = "GroupsCometMembersPageNewMembersSectionRefetchQuery"
doc_id = 6708203862579771

config = dotenv_values(".env")

friends_panel_class = "x9f619 x1n2onr6 x1ja2u2z x2bj2ny x1qpq9i9 xdney7k xu5ydu1 xt3gfkd xh8yej3 x6ikm8r x10wlt62 xquyuld"
friends_panel_selector = f"div[class='{friends_panel_class}']"

friend_link_class = (
    "x1i10hfl xjbqb8w x6umtig x1b1mbwd xaqea5y xav7gou x9f619 x1ypdohk xt0psk2 "
    "xe8uvvx xdj266r x11i5rnm xat24cr x1mh8g0r xexx8yu x4uap5 x18d9i69 xkhd6sd "
    "x16tdsg8 x1hl2dhg xggy1nq x1a2a7pz x1heor9g xt0b8zv"
)
friend_link_selector = f"a[class='{friend_link_class}']"

loading_friends_panel_class = "x1a2a7pz x78zum5 x1q0g3np x1a02dak x1qughib"
loading_friends_panel_selector = f"div[class='{loading_friends_panel_class}'][data-visualcompletion='loading-state']"

# Functions:

class Scraper:

    def __init__(self, group_id, members, friends_all=None):
        """
        Create the scraper.

        :param group_id: The Facebook id of the group to scrape
        :param members: An existing list of facebook group members
        :param friends: An existing dictionary of members mapped to their friends
        """
        self.group_id = group_id

        # Fetch the profile path
        path = fetch_profile_path()

        # Create path
        options = Options()
        options.set_preference('profile', r"" + path)

        # Create service
        service = Service(GeckoDriverManager().install())

        # Create the driver
        self.driver = Firefox(service=service, options=options)

        # Save given data
        self.members = members
        if friends_all is None:
            friends_all = {}
        self.friends_all = friends_all

    def login(self):
        """
        Login to Facebook using the Selenium webdriver.
        """
        # Open Facebook
        self.driver.get("https://www.facebook.com/")

        # Accept cookies
        accept_cookies_button = WebDriverWait(self.driver, 10).until(
            EC.element_to_be_clickable(
                (By.CSS_SELECTOR, "button[data-testid='cookie-policy-manage-dialog-accept-button']")
            )
        )

        accept_cookies_button.click()

        # Fill the login fields with our user and password.
        user_css_selector = "input[name='email']"
        password_css_selector = "input[name='pass']"

        username_input = WebDriverWait(self.driver, 10).until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, user_css_selector))
        )
        password_input = WebDriverWait(self.driver, 10).until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, password_css_selector))
        )

        username_input.clear()
        username_input.send_keys(config["user"])
        password_input.clear()
        password_input.send_keys(config["password"])

        # Click the login button
        time.sleep(1)
        WebDriverWait(self.driver, 2).until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, "button[type='submit']"))
        ).click()

    def scrape(self):
        """
        Scrape all members of a Facebook group using the Selenium webdriver.

        :return: the array of group members
        """
        # Navigate to the group's member list
        referrer = base_referrer.replace('{GROUP_ID}', group_id)

        time.sleep(2)
        self.driver.get(referrer)

        # Create session
        session = requests.session()
        session.cookies.update({
            cookie["name"]: cookie["value"]
            for cookie in self.driver.get_cookies()
        })

        pattern = r'\["DTSGInitData",\[\],{"token":"\S+","async_get_token":"\S+?"},\d+\]'
        match = re.search(pattern, self.driver.page_source)
        fb_dtsg_token = json.loads(match.group())[2]["token"]

        headers = {
            "accept": "*/*",
            "accept-language": "en-GB,en;q=0.5",
            "content-type": "application/x-www-form-urlencoded",
            "sec-ch-ua": "\" Not;A Brand\";v=\"99\", \"Google Chrome\";v=\"91\", \"Chromium\";v=\"91\"",
            "sec-ch-ua-mobile": "?0",
            "sec-fetch-dest": "empty",
            "sec-fetch-mode": "cors",
            "sec-fetch-site": "same-origin",
            "x-fb-friendly-name": query,
            "referrer": referrer,
            "referrerPolicy": "strict-origin-when-cross-origin",
        }

        page_info = dict(has_next_page=True, end_cursor=None)
        members = []

        # Request friends until there are no more
        page = 1
        while page_info["has_next_page"]:
            num_members = len(members)
            print(f"Reading group page {page}. ({num_members} members found)")

            response = session.post(
                url,
                headers=headers,
                data=urllib.parse.urlencode(
                    {
                        "fb_dtsg": fb_dtsg_token,
                        "fb_api_req_friendly_name": query,
                        "variables": json.dumps(
                            {
                                "count": 10,
                                "cursor": page_info["end_cursor"],
                                "groupID": group_id,
                                "id": group_id,
                                "recruitingGroupFilterNonCompliant": False,
                                "scale": 1.5,
                            }
                        ).replace(" ", ""),
                        "doc_id": doc_id,
                    }
                )
            )

            response_dict = json.loads(response.content)
            member_objects = response_dict["data"]["node"]["new_members"]["edges"]

            members += [
                dict(
                    id=str(member["node"]["id"]),
                    name=member['node']['name'],
                    url=member['node']['url']
                )
                for member in member_objects
                if member["node"]["__typename"] == "User"
            ]

            page_info = response_dict["data"]["node"]["new_members"]["page_info"]
            page += 1

        num_members = len(members)
        print(f"Number of members: {num_members}.")

        # Save members to the file
        with open("members", "w") as outfile:
            json.dump(members, outfile)

        self.members = members
        return members

    def crawl(self, start_pos=0, end_pos=None):
        """
        Scrape the friends for all members on Facebook using the Selenium webdriver.

        :param start_pos: The position of the group member to start with
        :param end_pos: The position of the group member to end with
        :return: the dictionary of members mapped to their friends
        """
        def visit_member_page(member):
            """
            Visit the url of a member.

            :param member: The member data
            """
            link = member["url"]
            url_parsed = urllib.parse.urlparse(link)

            if url_parsed.path == "/profile.php":
                member_link =  f"{link}&sk=friends"
            else:
                member_link = f"{link}/friends"

            self.driver.get(member_link)
            time.sleep(1.5)

        def wait_for_every_friend_to_load():
            loading_element = self.driver.find_elements_by_css_selector(
                loading_friends_panel_selector
            )

            while len(loading_element) > 0:
                self.driver.find_element_by_xpath('//body').send_keys(Keys.END)
                time.sleep(0.5)
                loading_element = self.driver.find_elements_by_css_selector(
                    loading_friends_panel_selector
                )

        def get_friend_by_url(url):
            for member in self.members:
                if member["url"] == url:
                    return member

            return None

        friends_all = self.friends_all
        # Add onto the friends
        num_members = len(self.members)
        record = self.members[start_pos:end_pos]
        for i, member in enumerate(record, start=1):
            print(f"Reading friends of {member['name']}. ({start_pos + i} of {num_members})")

            # Check this member
            visit_member_page(member)

            wait_for_every_friend_to_load()

            friends_panel = self.driver.find_element_by_css_selector(
                friends_panel_selector
            )

            friend_links = friends_panel.find_elements_by_css_selector(
                friend_link_selector
            )

            friends = []
            for link in friend_links:
                # Use the friend's url as a unique identifier
                link = link.get_attribute("href")
                friend = get_friend_by_url(link)

                # Check if this friend is in the group
                if friend is not None:
                    friends.append(
                        dict(
                            id=friend["id"],
                            name=friend["name"],
                            url=link,
                        )
                    )

            friends_all[member["id"]] = friends

            # Save current friends to the file
            with open("friends", "w") as outfile:
                json.dump(friends_all, outfile)

        self.friends_all = friends_all
        return friends_all


#! /usr/bin/env python3
"""
Replace predicted models with Web Apollo models accoriding to attribute, relace=
"""
import sys
import re
import logging
from gff3tool.lib import replace_OGS
from gff3tool.lib.gff3 import Gff3
from gff3tool.lib.utils import remove_files_from_list
import gff3tool.bin.gff3_sort as gff3_sort

def main(gff_file1, gff_file2, output_gff, report_fh, user_defined1=None, user_defined2=None, logger=None):
    logger_null = logging.getLogger(__name__+'null')
    null_handler = logging.NullHandler()
    logger_null.addHandler(null_handler)

    if not logger:
        logger = logger_null
    logger.info('Sorting the WA gff by following the order of Scaffold number and coordinates...')
    gff3_sort.main(gff_file1, output='WA_sorted.gff', logger=logger)

    logger.info('Sorting the other gff by following the order of Scaffold number and coordinates...')
    gff3_sort.main(gff_file2, output='other_sorted.gff', logger=logger)

    logger.info('Reading WA gff3 file...')
    gff3 = Gff3(gff_file='WA_sorted.gff', logger=logger_null)

    logger.info('Reading the other gff3 file...')
    gff3M = Gff3(gff_file='other_sorted.gff', logger=logger_null) #Maker

    logger.info('Identifying types of replacement based on replace tag...')
    ReplaceGroups = replace_OGS.Groups(WAgff=gff3, Pgff=gff3M, outsideNum=1, user_defined1=user_defined1, user_defined2=user_defined2, logger=logger_null)

    logger.info('Replacing...')
    u_types = set()
    u1_types = set()
    if user_defined1 is not None:
        for line in user_defined1:
            u1_types.add(line[0])
        u_types |= u1_types
    else:
        u1_types = None
    u2_types = set()
    if user_defined2 is not None:
        for line in user_defined2:
            u2_types.add(line[0])
        u_types |= u2_types
    else:
        u2_types = None
    roots = []
    transcripts = []
    unique = set()
    for line in gff3.lines:

        if user_defined1 is None:
            try:
                if line['line_type'] == 'feature' and 'Parent' not in line['attributes']:
                    roots.append(line)
            except:
                pass
        else:
            if line['type'] in u1_types:
                transcripts.append(line)
                for root in gff3.collect_roots(line):
                    if root['line_raw'] not in unique:
                        roots.append(root)
                        unique.add(root['line_raw'])

    #roots = [line for line in gff3.lines if line['line_type'] == 'feature' and 'Parent' not in line['attributes']]
    rnum, cnum, changed = 0, 0, 0
    cal_type_children = {}
    changed_rootid = set()
    not_orphan = set()
    for root in roots:
        rnum += 1
        if user_defined1 is None:
            children = root['children']
        else:
            children = []
            unique = set()
            if root['type'] in u1_types:
                children.append(root)
            else:
                for child in gff3.collect_descendants(root):
                    if child['type'] in u1_types:
                        if child['line_raw'] not in unique:
                            children.append(child)
                            unique.add(child['line_raw'])
            children = sorted(children, key=lambda k: k['line_index'])

        tags = {}
        cnum += len(children)
        maxisoforms = 0
        for child in children:
            tags[str(child['attributes']['replace'])] = 0
            for tag in child['attributes']['replace']:
                if not tag == 'NA':
                    not_orphan.add(tag)
                    t = gff3M.features[ReplaceGroups.mapName2ID[tag]][0]
                    if user_defined2 is None:
                        tmp = len(t['parents'][0][0]['children'])
                    else:
                        if len(t['parents']) == 0 and t['type'] in u2_types:
                            #this transcript don't have parent feature(e.g. gene), set the number of isoform as 1.
                            tmp = 1
                        else:
                            tmp = len(t['parents'][0][0]['children'])

                    if tmp > maxisoforms:
                        maxisoforms = tmp
        if len(tags) <= 1:
            if maxisoforms >= 2:
                root['attributes']['replace_type'] = 'multi-ref'
                for child in children:
                    child['attributes']['replace_type'] = 'multi-ref'
                if user_defined1 is None:
                    ans = ReplaceGroups.replacer_multi(root, ReplaceGroups, gff3M, u1_types, u2_types)
                else:
                    ans = ReplaceGroups.replacer_multi(root, ReplaceGroups, gff3M, u1_types, u2_types, gff3)
                report_fh.write('{0:s}\n'.format(ans))
                changed_rootid.add(root['attributes']['ID'])
                changed += 1
            else:
                ReplaceGroups.replacer(root, ReplaceGroups, gff3M, u1_types, gff3)
                changed_rootid.add(root['attributes']['ID'])
                changed += 1
        else:
            logger.info('[Warning] multiple replace tags in multiple isoforms! {0:s}. This model is not processed\n'.format(root['attributes']['ID']))
            report_fh.write('[Warning] multiple replace tags in multiple isoforms! {0:s}. This model is not processed\n'.format(root['attributes']['ID']))
        for child in children:
            if 'status' in child['attributes'] and (child['attributes']['status'] == 'Delete' or child['attributes']['status'] == 'delete'):
                child['attributes']['replace_type'] = 'Delete'
            if child['attributes']['replace_type'] in cal_type_children:
                cal_type_children[child['attributes']['replace_type']] += 1
            else:
                cal_type_children[child['attributes']['replace_type']] = 1

    cal_type = {}
    for i in ReplaceGroups.info:
        tokens = i.split('\t')
        tmp = re.search('(.+?):(.*)', tokens[3])
        if tmp.groups()[0] in cal_type:
            cal_type[tmp.groups()[0]] += 1
        else:
            cal_type[tmp.groups()[0]] = 1
        #print('{0:s}'.format(i))

    report_fh.write('# Number of WA loci: {0:d}\n'.format(rnum))
    report_fh.write('# Number of WA transcripts: {0:d}\n'.format(cnum))
    report_fh.write('# Number of WA loci that were used to replace the models in reference gff: {0:d}\n'.format(changed))

    for k, v in cal_type.items():
        if k == 'simple':
            report_fh.write('# Number of loci with {0:s}/Delete replacement: {1:d}\n'.format(k, v) )
        else:
            report_fh.write('# Number of loci with {0:s} replacement: {1:d}\n'.format(k, v) )
    for k, v in cal_type_children.items():
        report_fh.write('# Number of transcripts with {0:s} replacement: {1:d}\n'.format(k, v) )

    report_fh.write('Change_log\tOriginal_gene_name\tOriginal_transcript_ID\tOriginal_transcript_name\tTmp_OGSv0_ID\n')

    roots = []
    transcripts = []
    unique = set()
    for line in gff3M.lines:
        if user_defined2 is None:
            try:
                if line['line_type'] == 'feature' and 'Parent' not in line['attributes']:
                    roots.append(line)
            except:
                pass
        else:
            if line['type'] in u_types:
                transcripts.append(line)
                for root in gff3M.collect_roots(line):
                    if root['line_raw'] not in unique:
                        roots.append(root)
                        unique.add(root['line_raw'])

    # roots = [line for line in gff3M.lines if line['line_type'] == 'feature' and 'Parent' not in line['attributes']]
    for root in roots:
        if root['attributes']['ID'] not in changed_rootid:
            if user_defined2 is None:
                children = root['children']

            else:
                children = []
                unique = set()
                if root['type'] in u_types:
                    children.append(root)
                else:
                    for child in gff3M.collect_descendants(root):
                        if child['type'] in u_types:
                            if child['line_raw'] not in unique:
                                children.append(child)
                                unique.add(child['line_raw'])
                children = sorted(children, key=lambda k: k['line_index'])
        elif root['attributes']['ID'] in changed_rootid and user_defined1 is not None:
            children = []
            unique = set()
            if root['type'] in u1_types:
                children.append(root)
            else:
                for child in gff3.collect_descendants(root):
                    if child['type'] in u1_types:
                        if child['line_raw'] not in unique:
                            children.append(child)
                            unique.add(child['line_raw'])
            children = sorted(children, key=lambda k: k['line_index'])
        else:
            children = root['children']

        for child in children:
            cflag = 0
            if not child['line_status'] == 'removed':
                #print(child['attributes'])
                if 'replace_type' in child['attributes']:
                    for i in root['attributes']['replace']:
                        tname, tid, gid, tmpid = 'NA', 'NA', 'NA', 'NA'
                        tmpid = child['attributes']['ID']
                        if not i == 'NA':
                            t = gff3M.features[ReplaceGroups.mapName2ID[i]][0]
                            try:
                                tname = t['attributes']['Name']
                            except:
                                tname = t['attributes']['ID']
                            tid = t['attributes']['ID']
                            gid_list = list()
                            if user_defined2 is None:
                                for tp_line in t['parents']:
                                    for tp in tp_line:
                                        gid_list.append(tp['attributes']['ID'])
                                gid = ','.join(gid_list)
                            else:
                                for tp in gff3M.collect_roots(t):
                                    gid_list.append(tp['attributes']['ID'])
                                gid = ','.join(gid_list)
                            if tname not in not_orphan:
                                tmpid = 'NA'
                        report_fh.write('{0:s}\t{1:s}\t{2:s}\t{3:s}\t{4:s}\n'.format(ReplaceGroups.mapType2Log[child['attributes']['replace_type']], gid, tid, tname, tmpid))
                    del child['attributes']['replace_type']
                    cflag += 1
                if 'replace' in child['attributes']:
                    del child['attributes']['replace']
                if cflag == 0:
                    gid = None
                    gid_list = list()
                    if user_defined2 is None:
                        for p_line in child['parents']:
                            for p in p_line:
                                gid_list.append(p['attributes']['ID'])
                    else:
                        for p in gff3M.collect_roots(child):
                            gid_list.append(p['attributes']['ID'])

                    gid = ','.join(gid_list)
                    report_fh.write('{0:s}\t{1:s}\t{2:s}\t{3:s}\t{4:s}\n'.format(ReplaceGroups.mapType2Log['other'], gid, child['attributes']['ID'], ReplaceGroups.id2name[child['attributes']['ID']], child['attributes']['ID']))
            else:
                if 'status' in child['attributes'] and child['attributes']['status'] == 'Delete':
                    for i in child['attributes']['replace']:
                        if i == 'NA':
                            sys.exit('The replace tag for Delete replacement cannot be NA: {0:s}'.format(child['line_raw']))
                        t = gff3M.features[ReplaceGroups.mapName2ID[i]][0]
                        tname = t['attributes']['Name']
                        tid = t['attributes']['ID']
                        gid_list = list()
                        if user_defined2 is None:
                            for tp_line in t['parents']:
                                for tp in tp_line:
                                    gid_list.append(tp['attributes']['ID'])
                        else:
                            for tp_line in gff3M.collect_roots(t):
                                gid_list.append(tp_line['attributes']['ID'])

                        gid = ','.join(gid_list)

                        report_fh.write('{0:s}\t{1:s}\t{2:s}\t{3:s}\t{4:s}\n'.format(ReplaceGroups.mapType2Log['Delete'], gid, tid, tname, "NA"))
                    if 'replace' in child['attributes']:
                        del child['attributes']['replace']
        for attr in ['replace', 'replace_type', 'modified_track']:
            if attr in root['attributes']:
                del root['attributes'][attr]

    ReplaceGroups.name2id(gff3M)
    gff3M.write(output_gff)
    rm_list = ['WA_sorted.gff', 'other_sorted.gff']
    remove_files_from_list(rm_list)


import time
import logging
import os
import sys
import concurrent
import nrgpy

import src.common.utilities.function_helper as util_func
import src.nrg_parser.mast.utilities.iea_json.nrg_config as nrg_config
from src.nrg_parser.mast.logger_model.nrg.reader.symphoniepro_txt_reader import (
    SymphonieProTxtReader,
)
from src.nrg_parser.mast.utilities.iea_json.convert_to_iea_json import create_iea_json

def main():
    """Creates IEA-task43 json from NRG Symphonie Pro logger data files
    SymPRO Desktop PRO must be installed. Please specify installation location in nrg_config.py
    Args:
        project_name(str): ex: Morgan-County
        local_path(str): local path directory where logger data files(.rld) are located

    Output: iea_task43.json file under local path
    """

    logging.info("IN execute - NRG Systems")
    project_name = sys.argv[1]
    local_path = sys.argv[2]

    # check if path exists

    if not os.path.isdir(local_path):
        logging.error(
            f"Local path {local_path} does not exists. Please confirm rld path directory."
        )
        return

    local_path_dir_converted = os.path.join(local_path, "converted")

    # converted folder doesn't exist, create
    if not os.path.isdir(local_path_dir_converted):
        os.mkdir(local_path_dir_converted)
    else:
        util_func.remove_files_in_dir(local_path_dir_converted)

    process_start = time.perf_counter()

    # convert rld binary - txt conversion
    convert_start = time.perf_counter()

    # get symphonie pro desktop app locations
    if nrg_config.SYMPHONIE_APP is not None:
        symphro_path = nrg_config.SYMPHONIE_APP
    else:
        symphro_path = "C:/Program Files (x86)/Renewable NRG Systems/SymPRO Desktop/SymPRODesktop.exe"

    # # convert .rld file to txt
    converter = nrgpy.local_rld(
        rld_dir=local_path, out_dir=local_path_dir_converted, sympro_path=symphro_path
    )
    converter.convert()

    convert_end = time.perf_counter()
    logging.warning(
        f"PERF: Conversion {round(convert_end - convert_start, 2)} second(s)"
    )

    # read and parse
    logger_config_dict_list, sensor_config_dict_list = read_and_parse(
        local_path_dir_converted
    )
    if len(logger_config_dict_list) > 0 and len(sensor_config_dict_list) > 0:
        create_iea_json(
            logger_config_dict_list,
            sensor_config_dict_list,
            local_path,
            "NRG SymphoniePro",
            project_name,
        )

    # remove all converted files after the run
    util_func.remove_files_in_dir(local_path_dir_converted)

    process_end = time.perf_counter()
    logging.warning(
        f"PERF: PROCESS EXEC OVERALL: {round(process_end - process_start, 2)} second(s)"
    )
    logging.info(f"Please find the output file in {local_path}/iea_task43.json")
    logging.info("OUT execute - NRG Systems")

def read_and_parse(local_path):
    # initialize data
    logger_config_dict_list = []
    sensor_config_dict_list = []

    files = os.listdir(local_path)
    # Filtering only txt files.
    filename_list = [
        f for f in files if os.path.isfile(local_path + "/" + f) and f.endswith(".txt")
    ]

    # use a with statement to ensure threads are cleaned up promptly
    with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
        # Start the load operations and mark each future with its filename
        future_to_file = {
            executor.submit(read, local_path, filename): filename
            for filename in filename_list
        }

        for future in concurrent.futures.as_completed(future_to_file):
            filename = future_to_file[future]
            try:
                data = future.result()
            except Exception as exc:
                print("%r generated an exception: %s" % (filename, exc))
            else:
                if data is not None and len(data.timeseries_data) > 0:
                    lmc_list = data.get_iea_logger_main_config()
                    if lmc_list is not None:
                        logger_config_dict_list.append(lmc_list)
                        sensor_config_dict_list.extend(data.get_iea_sensor_config())
                #     else:
                #         print(f'invalid file: {filename}')
                else:
                    print(f"Data file has empty dataset: {filename}. Skip!")

    return logger_config_dict_list, sensor_config_dict_list

def read(local_path, filename):
    txt_filepath = os.path.join(local_path, filename)
    parsed_data = SymphonieProTxtReader(txt_filepath)
    return parsed_data

if __name__ == "__main__":
    main()


# coding=utf-8
"""
    在windows下运行！！！
    使用当前文件请下载Python2.7的版本。
    安装Python之后，用IDLE打开当前文件，使用F5运行
    这是一个修改图片名字的小程序。
"""

import os

def rename():
    path = "C:\\Users\\lyfw2\\OneDrive\\Photos"         # 修改文件名所在的完整文件夹路径,请复制完整文件夹路径之后把有'\'的加多一个'\'
    files = os.listdir(path)                            # 获取path路径下的所有名字的列表

    for file in files:                                  # 循环获取文件名
        """
        # 这是一个修改完整的文件名的方法,如果需要请把引号去掉，再把引号下面的去掉
        if os.path.isdir(file):
            continue
        old_name = os.path.join(path, file)
        file_type = os.path.splitext(file)[1]
        name = 'XXX' + file_type
        new_name = os.path.join(path, name)
        os.rename(old_name, new_name)
        """
        # 判断这文件名前面5个字母是否与之匹配，不等于就跳过(如果要修改其他字数把5改成要修改的字数的个数)
        if file[0:5] != 'SPSCF':
            continue
        else:
            old_name = os.path.join(path, file)         # 合并文件的路径
            file_type = os.path.splitext(file)[1]       # 获取当前文件的后缀名或是扩展名
            filename = os.path.splitext(file)[0]        # 获取当前文件名
            name = 'CHD' + filename[5:] + file_type     # 需要修改的文件名（filename[5:]可以根据你要改的字数的个数修改数字5，跟判断的尾数要相同）
            new_name = os.path.join(path, name)         # 合并文件的路径
            os.rename(old_name, new_name)               # 修改名字！

# 此处不要删除。
if __name__ == '__main__':
    rename()


# import pandas library
import pandas as pd

# load sample csv file into dataframe and count rows loaded
df = pd.read_csv('E:/Projects/python-pandas/GHCND_sample_csv.csv')
print(f'There are {len(df)} lines in the dataset.')

# inspect dataset structure interpred by pandas
df.info()

# inspect dataset first 10 lines
print(df.head(5))

# projection relational operation
df_proj = df[['STATION_NAME', 'DATE', 'TMAX', 'TMIN']]

# inspect first 5 lines of the projected dataframe
df_proj.head(5)

# save
df_proj.to_csv('E:/Projects/python-pandas/GHCND_sample_projected.csv')
print(f'There are {len(df_proj)} lines in the dataset.')
print(f'There are {len(df_proj.columns)} columns in the dataset.')


import json
from agora.testing import *
from accounts.models import User

def assertions_crud(resource, user):
    """
    Flow:
    User can list empty resources set
    User can create resource
    User can list data set
    User can update resource
    User can delete resource
    """
    url = RESOURCES_CRUD[resource]['url']
    data = RESOURCES_CRUD[resource]['create_data']
    edit_data = RESOURCES_CRUD[resource]['edit_data']
    assert user.get(url).json() == []
    resp = user.post(url, data)
    assert resp.status_code == 201
    for key, value in data.iteritems():
        assert resp.json()[key] == value
    assert len(user.get(url).json()) == 1
    id = resp.json()['id']
    if edit_data:
        resp = user.patch(
            url + id + '/',
            json.dumps(edit_data),
            content_type='application/json')
        assert resp.status_code == 200
        for key, value in edit_data.iteritems():
            assert resp.json()[key] == value

    resp = user.delete(url + id + '/')
    assert resp.status_code == 204
    assert user.get(url).json() == []

# Tests for resources with no foreign keys

def test_target_users(superadmin):
    assertions_crud('target_users', superadmin)

def test_providers(superadmin):
    assertions_crud('providers', superadmin)

# Tests for ServiceAdminship

# def test_serviceadminship_create(superadmin):

    # service_url = RESOURCES_CRUD['services']['url']
    # service_data = RESOURCES_CRUD['services']['create_data']
    # resp = superadmin.post(service_url, service_data)

    # service_id = resp.json()['id']
    # test_user, created = User.objects.get_or_create(
        # username='test_user',
        # email='test_user@test.org',
        # role='serviceadmin')
    # test_user.set_password('12345')
    # test_user.save()
    # sa_url = RESOURCES_CRUD['service_admins']['url']

    # """
    # Superadmin creates ServiceAdminship with status 'approved'.
    # Superadmin can delete a serviceAdminship
    # """
    # resp = superadmin.post(sa_url,
                           # {'admin': test_user.id, 'service': service_id})
    # sa_id = resp.json()['id']
    # assert resp.status_code == 201
    # assert resp.json()['state'] == 'approved'

    # resp = superadmin.delete(sa_url + sa_id + '/')
    # assert resp.status_code == 204

    # resp = superadmin.delete(service_url + service_id + '/')
    # assert resp.status_code == 204

    # resp = superadmin.post(service_url, service_data)
    # service_id = resp.json()['id']
    # test_user, created = User.objects.get_or_create(
        # username='test_user',
        # email='test_user@test.org')
    # """
    # Superadmin cannot create ServiceAdminship for user with roles 'observer',
    # 'superadmin' or 'admin'.
    # """
    # for role in ['observer', 'superadmin', 'admin']:
        # test_user.role = role
        # test_user.save()

        # resp = superadmin.post(sa_url,
                               # {'admin': test_user.id, 'service': service_id})
        # assert resp.status_code == 400

    # superadmin.delete(service_url + service_id + '/')

# def test_serviceadminship_update(superadmin):
    # """
    # Allowed state transitions are:
    # ('pending', 'approved'),
    # ('pending', 'rejected'),
    # ('rejected', 'pending'),
    # ('approved', 'pending'),
    # """

    # service_url = RESOURCES_CRUD['services']['url']
    # service_data = RESOURCES_CRUD['services']['create_data']
    # resp = superadmin.post(service_url, service_data)

    # service_id = resp.json()['id']
    # test_user, created = User.objects.get_or_create(
        # username='test_user',
        # email='test_user@test.org',
        # role='serviceadmin')
    # test_user.set_password('12345')
    # test_user.save()
    # sa_url = RESOURCES_CRUD['service_admins']['url']

    # resp = superadmin.post(sa_url,
                           # {'admin': test_user.id, 'service': service_id})
    # sa_id = resp.json()['id']

    # resp = superadmin.patch(
        # sa_url + sa_id + '/',
        # json.dumps({'state': 'rejected'}),
        # content_type='application/json')
    # assert resp.status_code == 400

    # resp = superadmin.patch(
        # sa_url + sa_id + '/',
        # json.dumps({'state': 'approved'}),
        # content_type='application/json')
    # assert resp.status_code == 400

    # resp = superadmin.patch(
        # sa_url + sa_id + '/',
        # json.dumps({'state': 'pending'}),
        # content_type='application/json')
    # assert resp.status_code == 200

# Tests for resources with related data


import secrets
from collections.abc import Iterable

from pydantic import Field
from pydantic_settings import BaseSettings

from metldata.accession_registry.accession_store import AccessionStore

class AccessionRegistryConfig(BaseSettings):
    """Config parameters and their defaults."""

    prefix_mapping: dict[str, str] = Field(
        ...,
        description="Specifies the ID prefix (values) per resource type (keys).",
        examples=[
            {
                "file": "GHGAF",
                "experiment": "GHGAE",
                "sample": "GHGAS",
                "dataset": "GHGAD",
            }
        ],
    )

    suffix_length: int = Field(8, description="Length of the numeric ID suffix.")

class AccessionRegistry:
    """Main class handling the accession registry."""

    class UnkownResourceTypeError(RuntimeError):
        """Raised when a resource type is specified that is unkown."""

        def __init__(self, *, specified_type: str, expected_types: Iterable[str]):
            """Specify the given as well as the expected resource types."""
            message = (
                f"The specified resource type '{specified_type} is unkown'."
                + " Expected one of: "
                + ", ".join(expected_types)
            )
            super().__init__(message)

    class AccessionGenerationError(RuntimeError):
        """Raised when a the generation of a new accession failed."""

    def __init__(
        self, *, config: AccessionRegistryConfig, accession_store: AccessionStore
    ):
        """Initialize with config."""
        self._config = config
        self._accession_store = accession_store

    def _assert_resource_type_exists(self, *, resource_type: str) -> None:
        """Checks whether the specified resource type is in the prefix mapping, raises
        and UnkownResourceTypeError otherwise.
        """
        if resource_type not in self._config.prefix_mapping:
            raise self.UnkownResourceTypeError(
                specified_type=resource_type,
                expected_types=self._config.prefix_mapping.keys(),
            )

    def _generate_accession(self, *, resource_type: str) -> str:
        """Generate a new accession."""
        self._assert_resource_type_exists(resource_type=resource_type)

        prefix = self._config.prefix_mapping[resource_type]
        suffix = "".join(
            str(secrets.randbelow(10)) for _ in range(self._config.suffix_length)
        )

        return prefix + suffix

    def get_accession(self, *, resource_type: str) -> str:
        """Generates and registers a new accession for a resource of the specified type."""
        for _ in range(10):
            # try 10 times to generate a new accession:
            accession = self._generate_accession(resource_type=resource_type)

            try:
                self._accession_store.save(accession=accession)
            except AccessionStore.AccessionAlreadyExistsError:
                continue

            return accession

        raise self.AccessionGenerationError(
            "Tried and failed 10 times to generate a new accession that is not used"
            + " already. The accession space might be exhausted."
        )


from PyQt5 import QtWidgets, QtCore, QtGui
import sys
import MainWindow as ui
import os

from Q1.Q1 import Question1
from Q2.Q2 import Question2
from Q3.Q3 import Question3

class Main(QtWidgets.QMainWindow, ui.Ui_MainWindow):
    def __init__(self):
        super().__init__()
        self.setupUi(self)

        # Load data
        self.pushButtonLoadFolder.clicked.connect(self.selectDir)
        self.pushButtonLoadImageL.clicked.connect(self.getImLPath)
        self.pushButtonLoadImageR.clicked.connect(self.getImRPath)

        # self.pushButtonFindCorners.clicked.connect(lambda: Q1Object.test(self.dirName))
        # Question 1
        self.pushButtonFindCorners.clicked.connect(lambda: Q1Object.findCorner(self.dirName))
        self.pushButtonFindIntrinsicMatrix.clicked.connect(lambda: Q1Object.findIntrinsic(self.dirName))
        self.pushButtonFindExtrinsicMatrix.clicked.connect(lambda: Q1Object.findExtrinsic(self.dirName, self.comboBoxFindExtrinsic.currentText()))
        self.pushButtonFindDistortionMatrix.clicked.connect(Q1Object.findDistortion)
        self.pushButtonShowUndistortedResult.clicked.connect(lambda: Q1Object.showUndistortion(self.dirName))

        # Question 2
        self.pushButtonShowWordsOnBoard.clicked.connect(lambda: Q2Object.onBoard(self.dirName, self.textEditAugmentedReality.toPlainText()))
        self.pushButtonShowWordsVertically.clicked.connect(lambda: Q2Object.verticalOnBoard(self.dirName, self.textEditAugmentedReality.toPlainText()))

        # Question 3
        self.pushButtonShowStereoDisparityMap.clicked.connect(lambda: Q3Object.stereoDisparityMap(self.ImLPath, self.ImRPath))

    def selectDir(self):
        self.dirName = QtCore.QDir.toNativeSeparators(QtWidgets.QFileDialog.getExistingDirectory(None, caption='Select a folder:', directory='C:\\', options=QtWidgets.QFileDialog.ShowDirsOnly))

    def selectFile(self):
        fileName = QtCore.QDir.toNativeSeparators(QtWidgets.QFileDialog.getOpenFileName(None, caption='Choose a File', directory='C:\\', filter='Image Files (*.png *.jpg *.bmp)')[0])  # get turple[0] which is file name
        return fileName

    def getImLPath(self):
        self.ImLPath = self.selectFile()

    def getImRPath(self):
        self.ImRPath = self.selectFile()

    # overide to force exit
    def closeEvent(self, a0: QtGui.QCloseEvent) -> None:
        super().closeEvent(a0)
        os._exit(0)

if __name__ == '__main__':
    app = QtWidgets.QApplication(sys.argv)
    Q1Object = Question1()
    Q2Object = Question2()
    Q3Object = Question3()
    window = Main()
    window.show()
    sys.exit(app.exec_())


#python3
#Steven 11/03/2020 image plot modoule

import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from commonModule.common import getRowAndColumn

def plotImagList2(imgList,nameList,gray=False,showTitle=True,showticks=True):
    nImg = len(imgList)
    nRow,nColumn = getRowAndColumn(nImg)

    #f, axarr = plt.subplots(nRow, nColumn,gridspec_kw = {'wspace':0, 'hspace':0}) #
    f, axarr = plt.subplots(nRow, nColumn, constrained_layout=True) #

    gs1 = gridspec.GridSpec(nRow, nColumn)
    gs1.update(wspace=0, hspace=0)

    print(len(f.axes),nRow,nColumn)

    for n in range(nImg):
        ax = plt.subplot(gs1[n])
        if 1:
            img = imgList[n]

            if showTitle:
                ax.title.set_text(nameList[n])
            if gray:
                ax.imshow(img,cmap="gray")
            else:
                ax.imshow(img)

            if not showticks:
                ax.set_yticks([])
                ax.set_xticks([])

            ax.margins(0, 0)
            #ax.xaxis.set_major_locator(plt.NullLocator())
            #ax.yaxis.set_major_locator(plt.NullLocator())

    #plt.grid(True)
    plt.tight_layout(pad=0)
    plt.subplots_adjust(wspace=0, hspace=0)
    #plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)
    plt.show()

def plotImagList(imgList,nameList,gray=False,showTitle=True,showticks=True):
    nImg = len(imgList)
    nRow,nColumn = getRowAndColumn(nImg)

    for n in range(nImg):
        img = imgList[n]
        ax = plt.subplot(nRow, nColumn, n + 1)
        if showTitle:
            ax.title.set_text(nameList[n])
        if gray:
            plt.imshow(img,cmap="gray")
        else:
            plt.imshow(img)

        if not showticks:
            ax.set_yticks([])
            ax.set_xticks([])

        ax.set_aspect('equal')
        #ax.margins(0, 0)
        #ax.xaxis.set_major_locator(plt.NullLocator())
        #ax.yaxis.set_major_locator(plt.NullLocator())
    #plt.grid(True)
    plt.tight_layout()
    #plt.subplots_adjust(wspace=0, hspace=0)
    #plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)
    plt.show()

def main():
    #file = r'./res/obama.jpg'#'./res/Lenna.png' #
    #img = loadImg(file,mode=cv2.IMREAD_GRAYSCALE) # IMREAD_GRAYSCALE IMREAD_COLOR
    #infoImg(img)
    #showimage(binaryImage2(img,thresHMin=50,thresHMax=150))
    pass

if __name__=='__main__':
    main()


def binary_search(A, k, high, low):
    while low <= high:
        mid = (high + low) // 2
        if A[mid] == k:
            return mid
        elif A[mid] > k:
            high = mid - 1
        elif A[mid] < k:
            low = mid + 1

    return -1

if __name__ == '__main__':
    n1A = [int(n1A) for n1A in input().split()]
    n1 = n1A[0]
    A = n1A[1:]
    n2K = [int(n2K) for n2K in input().split()]
    K = n2K[1:]
    for k in K:
        print(binary_search(A, k, n1-1, 0), end = ' ')


import argparse
import csv
import string

from operator import itemgetter
from glob import glob
from Bio import SeqIO
from random import randint

def seqs_under_peaks():

    ########################
    #command line arguments#
    ########################

    parser = argparse.ArgumentParser()

    #MANDATORY PARAMETERS
    parser.add_argument("genome",help="Full path to the reference genome file in fasta-format.",type=str)
    parser.add_argument("peaks",help="Full path to the file containing peaks in igv-format.",type=str)
    parser.add_argument("outname",help="Full path for the output file.",type=str)

    #OPTIONAL PARAMETERS
    parser.add_argument("-N","--numpeaks",help="How many of the top peaks are analyzed (default=100).",type=int,default=100)
    parser.add_argument("-w", "--peakwidth",help="Width of the sequence around the peak summit (w+1+w), deafault=25.",type=int,default=25)
    parser.add_argument("-s","--sorting",help="1=sort by read count, 2=sort by p-value, 3=random peaks, 4=sort by score,default=1",type=int,choices=[1,2,3,4])

    args = parser.parse_args()

    N = args.numpeaks
    w = args.peakwidth

    peaks = []
    #[[chr,index,binding site length,signal],...]

    #print "Reading in peaks...",
    #reading in the peaks
    with open(args.peaks,'rb') as peakfile:
        r = csv.reader(peakfile,delimiter='\t')
        for row in r:
            if row[0].count('chromosome')>0: continue
            chrom = row[0]
            start = int(float(row[1]))
            end = int(float(row[2]))
            if end-start>1: loc = start+(end-start)/2
            else: loc = start

            if args.sorting==1: signal = float(row[-2]) #THIS IS THE TOTAL NUMBER OF READS PER PEAK!
            else: signal = float(row[-1]) #THIS IS THE P-VALUE
            peaks.append([chrom,loc,end-start,signal])

    #selecting the required number of top peaks
    #print "total number of peaks="+str(len(peaks))+"..."
    if args.sorting==1: peaks = sorted(peaks,key=itemgetter(3),reverse=True)[:N]
    elif args.sorting==2: peaks = sorted(peaks,key=itemgetter(3),reverse=False)[:N]
    elif args.sorting==4: peaks = sorted(peaks,key=itemgetter(3),reverse=True)[:N]
    else:
        #we select random N peaks from the input
        new_peaks = []
        used = set()
        for i in range(0,N):
            r = randint(0,len(peaks)-1)
            while r in used: r = randint(0,len(peaks)-1)
            new_peaks.append(peaks[r])
            used.add(r)
        peaks = new_peaks

    #print "done!"

    #print "Reading in sequences...",
    sequences = []
    names = []

    #sorting found peaks according to chromosome
    top_peaks = {}
    #key = chromosome
    #value = [[loc1,len1],[loc2,len2],...]
    for i in range(0,len(peaks)):
        if peaks[i][0] not in top_peaks: top_peaks[peaks[i][0]] = [[peaks[i][1],peaks[i][2]]]
        else: top_peaks[peaks[i][0]].append([peaks[i][1],peaks[i][2]])

    #retrieving the underlying sequences from reference genome
    handle = open(args.genome,'rU')
    for record in SeqIO.parse(handle,"fasta"):
        chrom = record.id
        if chrom not in top_peaks: continue
        seq = record.seq
        #saving the sequence for each peak in this chromosome
        for peak in top_peaks[chrom]:
            #print peak
            sequences.append(seq[peak[0]-w:peak[0]+w])
            names.append(chrom+"_"+str(peak[0]))
    handle.close()
    #print "done!"

    #print "Saving the sequences...",
    f = open(args.outname,'w')
    for s in range(0,len(sequences)):
        f.write(">"+str(s)+"_"+str(names[s])+"\n"+str(sequences[s]).upper()+"\n")
    f.close()
    #print "done!"
#end

seqs_under_peaks()


'''
Decision Tree Induction
Starter code author: Steven Bogaerts
'''
import math

class AttributeSpec:
    def __init__(self,name,vals):
        self.__name = name
        self.__vals = vals
    def getName(self):
        return self.__name
    def getValAt(self,i):
        return self.__vals[i]
    def getIndexOf(self,value):
        return self.__vals.index(value)
    def getNumVals(self):
        return len(self.__vals)
    def __repr__(self):
        return "AttributeSpec{" + str(self.__name) + ", " + str(self.__vals) + "}"
def testAttributeSpec():
    print("==================== testAttributeSpec ====================")
    tempSpec = AttributeSpec("Temperature", ["Low", "Medium", "High"])
    ratingSpec = AttributeSpec("Rating", [0, 1, 2, 3, 4, 5])

    print(tempSpec)
    print(ratingSpec)

    print(tempSpec.getName(), tempSpec.getValAt(1), tempSpec.getIndexOf("Medium"), tempSpec.getNumVals())
class Example(object):
    '''
    Example is an example - a collection of attribute keys and values,
    and possibly the class of the example, if known.

    cls - the class of this example (can be left off if unknown)

    nameValDict - dictionary from attribute name (from an AttributeSpec) to the
                 corresponding value for this Example.
    '''

    def __init__(self, nameValDict, cls=None):
        self.__cls = cls
        self.__nameValDict = nameValDict
    def getClass(self):
        return self.__cls
    def hasSameClassAs(self,other):
        return self.__cls == other.__cls
    def getValFor(self,attributeName):
        return self.__nameValDict.get(attributeName)
    def __repr__(self):
        return "Example{"+ self.__cls + ", " + str(self.__nameValDict) + "}"
def testExample():
    print("==================== testExample ====================")
    # "Do you need help lifting this thing?"
    ex1 = Example({"height":5, "width":4, "isHeavy":False}, "No")
    ex2 = Example({"height":5, "width":12, "isHeavy":True}, "Yes")
    ex3 = Example({"height":5, "width":2, "isHeavy":True}) # cls not provided, so it's None (see __init__)

    print(ex1)
    print(ex1.getClass())
    print(ex1.hasSameClassAs(ex2))
    print(ex1.getValFor("width"))
class ExampleList(object):
    '''
    A list of Example objects.
    '''

    def __init__(self, exampleList):
        self.__exampleList = exampleList
    def __repr__(self):
        return "ExampleList{" + str(self.__exampleList) + "}"
    def getExampleAt(self, i):
        return self.__exampleList[i]
    def getNumExamples(self):
        return len(self.__exampleList)
    def isEmpty(self):
        return self.__exampleList == []
    def append(self, ex):
        self.__exampleList.append(ex)
    def allSameClass(self):
        standardCLS = self.__exampleList[0]
        for example in self.__exampleList:
            if not standardCLS.hasSameClassAs(example):
                return False
        return True
        def specialLog2(self, val):
            if val == 0:
                return 0
            else:
                return math.log(val, 2)
    def specialLog2(self, val):
        if val == 0:
            return 0
        else:
            return math.log(val, 2)
    def calcEntropy(self, classAttrSpec):
        # Calculates the entropy of this example list over the class
        if(self.isEmpty()):#special case if input list is empty
            return 0
        numberOfVals = classAttrSpec.getNumVals()
        uniqueValuesList = []*numberOfVals
        for x in range(numberOfVals):#get the unique outcomes names
            if not classAttrSpec.getValAt(x) in uniqueValuesList:
                uniqueValuesList.append(classAttrSpec.getValAt(x))
        numberOfOccurance = [0]*numberOfVals
        for x in range(self.getNumExamples()):#find how many occurence of each outcome there is
            for value in uniqueValuesList:
                if value == self.__exampleList[x].getClass():
                    numberOfOccurance[uniqueValuesList.index(value)] +=1
        totalEntrees = 0
        for number in numberOfOccurance:
            totalEntrees+=number
        entropy = 0
        for x in range(len(uniqueValuesList)):
            entropy+= -(numberOfOccurance[x]/totalEntrees) * self.specialLog2(numberOfOccurance[x]/totalEntrees)
        return entropy
    def calcGain(self, attrSpec, classAttrSpec):
        # Calculates the gain from splitting on attribute attrSpec.
        splitExamples = self.split(attrSpec)
        gain = 0
        numOfExamples = self.getNumExamples()
        for exampleList in splitExamples:
            numOfExamplesInExample = exampleList.getNumExamples()
            gain+=numOfExamplesInExample/numOfExamples * exampleList.calcEntropy(classAttrSpec)
        return self.calcEntropy(classAttrSpec) - gain
    def chooseAttr(self, attrSpecs, classAttrSpec):
        # Returns the attribute specification that best splits the example list.
        listOfGain = []
        for attrSpec in attrSpecs:
            listOfGain.append(self.calcGain(attrSpec,classAttrSpec))
        return attrSpecs[listOfGain.index(max(listOfGain))]
    def countClassValues(self, classAttrSpec):
        '''
        For the examples in this ExampleList,
        counts the number of occurrences of each class.
        Returns a list valCount, where valCount[i] is the number of occurrences of the ith class value
        (as ordered in classAttrSpec).
        '''
        # Initialize valCount
        valCount = [0 for i in range(classAttrSpec.getNumVals())]
        ''' The above line (a "list comprehension") does the same as the following:
        valCount = []
        for i in range(classAttrSpec.getNumVals()):
            valCount.append(0)
        '''

        for ex in self.__exampleList:
            for i in range(classAttrSpec.getNumVals()):
                if classAttrSpec.getValAt(i) == ex.getClass():
                    valCount[i] += 1

        return valCount
    def getMajorityClass(self, classAttrSpec):
        '''
        Determines the class that describes the majority of examples
        in the list.
        '''
        valCount = self.countClassValues(classAttrSpec)

        maxID = 0
        for i in range(1, len(valCount)):
            if valCount[i] > valCount[maxID]:
                maxID = i
        return classAttrSpec.getValAt(maxID) # change this to what it should be
    def split(self, attrSpec):
        '''
        Splits the examples by attribute.
        Returns a list of ExampleList objects. (So essentially, a list of lists, or a 2-D list.)
        '''

        # Initialize the splitExamples structure to a list of empty ExampleList objects.
        splitExamples = [ExampleList([]) for i in range(attrSpec.getNumVals())]
        ''' The above line (a "list comprehension") does the same as the following:
        splitExamples = []
        for i in range(attrSpec.getNumVals()):
            splitExamples.append(ExampleList([]))
        '''

        attrName = attrSpec.getName()
        for ex in self.__exampleList: # Using the list for-each notation
            for i in range(attrSpec.getNumVals()):
                if attrSpec.getValAt(i) == ex.getValFor(attrName):
                    splitExamples[i].append(ex)

        return splitExamples
def testExampleListBasic():
    print("==================== testExampleListBasic ====================")
    ex2 = Example({"height":5, "width":12, "isHeavy":True}, "Yes")
    ex3 = Example({"height":5, "width":2, "isHeavy":True}, "Yes")
    ex4 = Example({"height":12, "width":7, "isHeavy":True}, "Yes")

    exList = ExampleList([ex2, ex3])
    print("1)", exList)
    print("2)", exList.getExampleAt(1))
    print("3)", exList.getNumExamples())
    print("4)", exList.isEmpty())

    exList.append(ex4)
    print("5)", exList)
def testAllSameClass():
    print("==================== testAllSameClass ====================")
    ex2 = Example({"height":5, "width":12, "isHeavy":True}, "Yes")
    ex3 = Example({"height":5, "width":2, "isHeavy":True}, "Yes")
    ex4 = Example({"height":12, "width":7, "isHeavy":False}, "Yes")
    ex5 = Example({"height":9, "width":3, "isHeavy":False}, "No")
    ex6 = Example({"height":9, "width":3, "isHeavy":True}, "Yes")

    exList = ExampleList([ex2, ex3, ex4])
    print(exList.allSameClass())
    exList.append(ex5)
    exList.append(ex6)
    print(exList.allSameClass())
def testCountClassValues():
    print("==================== testCountClassValues ====================")
    ex2 = Example({"height":5, "width":12, "isHeavy":True}, "Yes")
    ex3 = Example({"height":5, "width":2, "isHeavy":True}, "Yes")
    ex4 = Example({"height":12, "width":7, "isHeavy":False}, "Yes")
    ex5 = Example({"height":9, "width":3, "isHeavy":False}, "No")
    ex6 = Example({"height":9, "width":3, "isHeavy":True}, "Yes")

    exList = ExampleList([ex2, ex3, ex4, ex5, ex6])
    classAttr = AttributeSpec("Need Help?", ["No", "Yes"])
    print(exList.countClassValues(classAttr))
def testGetMajorityClass():
    print("==================== testGetMajorityClass ====================")
    ex2 = Example({"height":5, "width":12, "isHeavy":True}, "Yes")
    ex3 = Example({"height":5, "width":2, "isHeavy":True}, "Yes")
    ex4 = Example({"height":12, "width":7, "isHeavy":False}, "Yes")
    ex5 = Example({"height":9, "width":3, "isHeavy":False}, "No")
    ex6 = Example({"height":9, "width":3, "isHeavy":True}, "Yes")

    exList = ExampleList([ex2, ex3, ex4, ex5, ex6])
    classAttr = AttributeSpec("Need Help?", ["No", "Yes"])
    print(exList.getMajorityClass(classAttr))

    ex7 = Example({"height":9, "width":4, "isHeavy":False}, "No")
    exList2 = ExampleList([ex5, ex6, ex7])
    print(exList2.getMajorityClass(classAttr))
def testSplit():
    print("==================== testSplit ====================")
    ex2 = Example({"height":'S', "width":'L', "isHeavy":True}, "Yes")
    ex3 = Example({"height":'S', "width":'S', "isHeavy":True}, "Yes")
    ex4 = Example({"height":'L', "width":'M', "isHeavy":False}, "Yes")
    ex5 = Example({"height":'M', "width":'S', "isHeavy":False}, "No")
    ex6 = Example({"height":'M', "width":'S', "isHeavy":True}, "Yes")

    exList = ExampleList([ex2, ex3, ex4, ex5, ex6])
    heightSpec = AttributeSpec("height", ['S', 'M', 'L'])
    widthSpec = AttributeSpec("width", ['S', 'M', 'L'])
    isHeavySpec = AttributeSpec("isHeavy", [False, True])

    print("----- Split by height")
    splitByHeight = exList.split(heightSpec)
    for exListID in range(len(splitByHeight)):
        print("Height", heightSpec.getValAt(exListID), "has list:", splitByHeight[exListID], end='\n\n')

    print("----- Split by isHeavy")
    splitByIsHeavy = exList.split(isHeavySpec)
    for exListID in range(len(splitByIsHeavy)):
        print("isHeavy", isHeavySpec.getValAt(exListID), "has list:", splitByIsHeavy[exListID], end='\n\n')
def testAll():
    testAttributeSpec()
    testExample()
    testExampleListBasic()
    testAllSameClass()
    testCountClassValues()
    testGetMajorityClass()
    testSplit()
############################################################################
def testEntropy():
    classAttr = AttributeSpec("needHelp", ["No", "Yes"])

    emptyList = ExampleList([])
    print(emptyList.calcEntropy(classAttr)) # The entropy of an empty list is defined to be 0, as a special case

    ex1 = Example({"height":5, "width":4, "isHeavy":False}, "No")
    ex2 = Example({"height":5, "width":12, "isHeavy":True}, "Yes")
    ex3 = Example({"height":5, "width":2, "isHeavy":True}, "Yes")
    ex4 = Example({"height":12, "width":7, "isHeavy":True}, "Yes")
    exList = ExampleList([ex1, ex2, ex3, ex4])
    print(exList.calcEntropy(classAttr))

    classAttr = AttributeSpec("result", [0, 1, 2])
    ex1 = Example({"a":2}, 1)
    ex2 = Example({"a":3}, 1)
    ex3 = Example({"a":5}, 2)
    ex4 = Example({"a":4}, 2)
    ex5 = Example({"a":1}, 0)
    exList = ExampleList([ex1, ex2, ex3, ex4, ex5])
    print(exList.calcEntropy(classAttr))
def testGain():
    heightAttr = AttributeSpec("height", [1, 2, 3])
    widthAttr = AttributeSpec("width", [1, 2, 3])
    heavyAttr = AttributeSpec("isHeavy", [False, True])
    classAttr = AttributeSpec("needHelp", ["No", "Yes"])
    ex1 = Example({"height":1, "width":1, "isHeavy":False}, "No")
    ex2 = Example({"height":1, "width":3, "isHeavy":True}, "Yes")
    ex3 = Example({"height":1, "width":2, "isHeavy":True}, "Yes")
    ex4 = Example({"height":3, "width":3, "isHeavy":True}, "Yes")
    ex5 = Example({"height":2, "width":2, "isHeavy":False}, "No")
    exList = ExampleList([ex1, ex2, ex3, ex4, ex5])

    print(exList.calcGain(heightAttr, classAttr))
    print(exList.calcGain(widthAttr, classAttr))
    print(exList.calcGain(heavyAttr, classAttr))
def testChooseAttr():
    heightAttr = AttributeSpec("height", [1, 2, 3])
    widthAttr = AttributeSpec("width", [1, 2, 3])
    heavyAttr = AttributeSpec("isHeavy", [False, True])
    classAttr = AttributeSpec("needHelp", ["No", "Yes"])
    ex1 = Example({"height":1, "isHeavy":False, "width":1}, "No")
    ex2 = Example({"height":1, "isHeavy":True, "width":3}, "Yes")
    ex3 = Example({"height":1, "isHeavy":True, "width":2}, "Yes")
    ex4 = Example({"height":3, "isHeavy":True, "width":3}, "Yes")
    ex5 = Example({"height":2, "isHeavy":False, "width":2}, "No")
    exList = ExampleList([ex1, ex2, ex3, ex4, ex5])

    print("height, heavy, width gains:")
    print(exList.calcGain(heightAttr, classAttr))
    print(exList.calcGain(heavyAttr, classAttr))
    print(exList.calcGain(widthAttr, classAttr))

    print("Attribute chosen:")
    print(exList.chooseAttr([heightAttr, heavyAttr, widthAttr], classAttr))
###########################################################################
class DTree(object):
    '''
    A DTree is a decision tree.

    The constructor (__init__) just initializes the fields to None.
    To actually make a DTree, though, call either makeLeaf or makeDTree instead.
    (Both of these call the constructor and then set certain fields.)

    A DTree has three fields:
    1) question is the AttributeSpec asked by the root of this tree
    2) children is a dictionary mapping question.vals to DTree objects
    3) cls is something in classAttrSpec.__vals - it's one of the valid class values

    One of the following must be true:
    - The tree is just a leaf. So cls has the classification, while children and question are None.
    - The tree is not a leaf. So children and question are set, while cls is None.
    '''

    def __init__(self):
        '''
        Just "declares" the fields and initializes them to None. They get set
        in one of the class methods.
        '''
        self.question = None # The AttributeSpec asked by the root of this tree
        self.children = None # dictionary mapping question.__vals to DTree objects

        self.cls = None # A classification - one of the values in classAttrSpec.__vals

    @classmethod
    def makeLeaf(pythonClass, classification):
        '''
        Note the @classmethod annotation. This is the same as a static method in Java.
        Call it like this:
            DTree.makeLeaf(some value in classAttrSpec.vals)
        When you do, pythonClass will have the value DTree. (This is similar to non-class methods,
        in which self takes on the value of the object you called it on.)
        The single argument you pass to makeLeaf will be stored in the classification
        formal parameter.

        The result: we return a DTree that is a leaf, assigning the given classification.
        '''
        t = pythonClass()       # construct a DTree object (calls __init__); fields initialized to None
        # Making a leaf, so self.children and self.question stay None
        t.cls = classification  # store the classification for this leaf
        return t

    @classmethod
    def makeDTree(pythonClass, question, children):
        '''
        Call this method:  DTree.makeDTree(question asked at root, children DTrees)
        to make a DTree that is not a leaf.
        '''
        t = pythonClass() # construct a DTree object (calls __init__); fields initialized to None
        t.question = question # store the question asked at this node
        t.children = children # store the child DTrees. Maps t.question.vals to DTrees.
        # Making an internal node (not a leaf), so self.cls stays None
        return t

    def isLeaf(self):
        return self.children == None

    # TO DO
    def classify(self, example):
        '''
        Given this DTree, determine the classification of the given example.
        '''
        if self.isLeaf():
            return self.cls
        return self.children[example.getValFor(self.question.getName())].classify(example)

    def __repr__(self):
        return self.__reprHelper("")

    def __reprHelper(self, spacing):
        if self.isLeaf():
            return spacing + str(self.cls) + "\n"
        else:
            result = spacing + "[" + str(self.question.getName()) + "\n"

            newSpacing = spacing + "    "
            for ans, child in self.children.items():
                if child.isLeaf():
                    result += newSpacing + str(ans) + " -> " + str(child.__reprHelper(""))
                else:
                    result += newSpacing + str(ans) + " :\n  " + str(child.__reprHelper(newSpacing))

            result += spacing + "  ]\n"
            return result

    def __makeSpacing(self, n):
        result = ""
        for i in range(n):
            result += " "
        return result
########################################################################
def demoTree(verbose=True):
    '''
    Shows an example of creating a DTree manually (not by induction)
    and using it to classify examples.
    '''

    # Define the attributes that will be used in this domain.
    attrSpecs = [AttributeSpec("Sunny?", [True, False]),
                 AttributeSpec("Warm?", ["Y", "N"]),
                 AttributeSpec("Age?", ["<20", "20-40", ">40"])]

    # Define the class - the question we're asking of each example.
    classAttrSpec = AttributeSpec("Swim?", ["Y", "M", "N"])
    defaultClass = "M"

    '''
    Build this tree manually:
                      Sunny?
               True            False
             Warm?               ans: N
          Y       N
        ans: Y    ans: N
    '''
    warmLeftLeaf = DTree.makeLeaf('Y') # This leaf is ans: Y
    warmRightLeaf = DTree.makeLeaf('N') # This leaf is ans: N

    # First arg: the question asked (Warm)
    # Second arg: a dictionary (note the { key:value, ...} notation) mapping from answer-to-Warm? to a tree (or leaf)
    warmTree = DTree.makeDTree(attrSpecs[1], {'Y':warmLeftLeaf, 'N':warmRightLeaf})

    sunnyRight = DTree.makeLeaf('N') # This leaf is ans: N
    tree = DTree.makeDTree(attrSpecs[0], {True : warmTree, False : sunnyRight})

    if (verbose):
        print("Root question:", tree.question, end='\n\n')
        print("Root children (dictionary from question answer to tree):", tree.children, sep='\n', end='\n\n')
        print("Root, one of the children:", tree.children.get(True), sep='\n', end='\n\n')
        print("Root, another of the children:", tree.children.get(False), sep='\n', end='\n\n')
        print("Root class:", tree.cls, end='\n\n') # None, because root is not a leaf

        print("Returning:")
    return tree
def testClassify():
    tree = demoTree(False) # get the tree built above
    print(tree)

    # Set up the list of examples we want to classify with the tree.
    # Here, the actual classification of the examples is also known,
    # so we can check the tree's performance against the actual
    # classifications.
    examples = ExampleList([Example({"Sunny?" : True,
                                     "Warm?" : "Y",
                                     "Age?" : "<20"},
                                    "Y"),
                            Example({"Sunny?" : True,
                                     "Warm?" : "Y",
                                     "Age?" : "20-40"},
                                    "Y"),
                            Example({"Sunny?" : True,
                                     "Warm?" : "N",
                                     "Age?" : "20-40"},
                                    "N"),
                            Example({"Sunny?" : False,
                                     "Warm?" : "N",
                                     "Age?" : "20-40"},
                                    "N"),
                            Example({"Sunny?" : False,
                                     "Warm?" : "Y",
                                     "Age?" : "20-40"},
                                    "N")
                            ])

    # Compare what the tree says (ans) with the actual classification (actual)
    for exID in range(examples.getNumExamples()):
        ex = examples.getExampleAt(exID)
        print("Ans: " + str(tree.classify(ex)) + "     Actual: " + str(ex.getClass()))

########################################################################

# TO DO
def decisionTreeLearning(examples, attrSpecs, classAttrSpec, defaultClass):
    '''
    Given a set of examples, attribute specifications, a class attribute specification,
    and a default class, induce a decision tree on the examples.
    Returns that tree.

    examples - an ExampleList object
    attrSpecs - a list of AttributeSpec instances
    classAttrSpec - an AttributeSpec instance, representing the class
    defaultClass - something in classAttrSpec.__vals
    '''
    if examples.isEmpty():
        return defaultClass
    elif examples.allSameClass():
        return examples.getExampleAt(0).getClass()
    elif attrSpecs == None:
        return examples.getMajorityClass(classAttrSpec)
    else:
        maj = examples.getMajorityClass(classAttrSpec)
        attr = examples.chooseAttr(attrSpecs,classAttrSpec)
        root =  DTree.makeDTree(attr, {})
        for vi in range(attr.getNumVals()):
            Si =  examples.split(attr)[vi]
            A = attrSpecs.copy()
            A.remove(attr)
            subtree = decisionTreeLearning(Si, A , classAttrSpec, maj)
            if type(subtree) == DTree:
                root.children[attr.getValAt(vi)] = subtree
            else:
                root.children[attr.getValAt(vi)] = DTree.makeLeaf(subtree)
        return root

########################################################################
def runTest(attrSpecs, classAttrSpec, defaultClass, examples):
    # Induce the decision tree.
    tree = decisionTreeLearning(examples, attrSpecs, classAttrSpec, defaultClass)
    print(tree)

    # Run the decision tree on the training set.
    for exID in range(examples.getNumExamples()):
        ex = examples.getExampleAt(exID)
        print("Ans: " + str(tree.classify(ex)) + "     Actual: " + str(ex.getClass()))

########################################################################
def smallTest():
    '''
    A small example of decision tree induction.
    "Is it a good day to go swimming?"
    '''

    # Define the attribute specifications.
    attrSpecs = [AttributeSpec("Sunny?", [True, False]),
                 AttributeSpec("Warm?", ["Y", "N"]),
                 AttributeSpec("Age?", ["<20", "20-40", ">40"])]

    # Define the class attribute specification.
    classAttrSpec = AttributeSpec("Swim?", ["Y", "M", "N"])
    defaultClass = "M"

    # Define the examples on which to induce the decision tree.
    examples = ExampleList([Example({"Sunny?" : True,
                                     "Warm?" : "Y",
                                     "Age?" : "<20"},
                                    "Y"),
                            Example({"Sunny?" : True,
                                     "Warm?" : "Y",
                                     "Age?" : "20-40"},
                                    "Y"),
                            Example({"Sunny?" : True,
                                     "Warm?" : "N",
                                     "Age?" : "20-40"},
                                    "N")
                            ])

    runTest(attrSpecs, classAttrSpec, defaultClass, examples)

########################################################################
def xorTest():
    # Define the attribute specifications.
    attrSpecs = [AttributeSpec("A", ['F', 'T']),
                 AttributeSpec("B", ['F', 'T'])]
    classAttrSpec = AttributeSpec("A XOR B?", [0, 1])
    defaultClass = 'Y'

    # Define the examples.
    e1 = Example({'A':'F',
                  'B':'F',},
                 0)
    e2 = Example({'A':'F',
                  'B':'T',},
                 1)
    e3 = Example({'A':'T',
                  'B':'F',},
                 1)
    e4 = Example({'A':'T',
                  'B':'T',},
                 0)
    examples = ExampleList([e1, e2, e3, e4])

    runTest(attrSpecs, classAttrSpec, defaultClass, examples)

#########################################################################
def rnTest():
    '''
    This is a larger example, from Russell and Norvig chapter 18.
    "Should I wait for a table at this restaurant tonight?"
    '''

    # Define the attribute specifications.
    attrSpecs = [AttributeSpec("alt", ['n', 'y']),
                 AttributeSpec("bar", ['n', 'y']),
                 AttributeSpec("fri", ['n', 'y']),
                 AttributeSpec("hun", ['n', 'y']),
                 AttributeSpec("pat", ['none', 'some', 'full']),
                 AttributeSpec("price", ['$', '$$', '$$$']),
                 AttributeSpec("rain", ['n', 'y']),
                 AttributeSpec("res", ['n', 'y']),
                 AttributeSpec("type", ['french', 'italian', 'thai', 'burger']),
                 AttributeSpec("est", ['b010', 'b1030', 'b3060', 'g60'])]
    classAttrSpec = AttributeSpec("Will wait", [False, True])
    defaultClass = False

    # Define the examples.
    e1 = Example({'alt': 'y',
                  'bar': 'n',
                  'fri': 'n',
                  'hun': 'y',
                  'pat': 'some',
                  'price': '$$$',
                  'rain': 'n',
                  'res': 'y',
                  'type': 'french',
                  'est': 'b010'},
                 True)

    e2 = Example({'alt': 'y',
                  'bar': 'n',
                  'fri': 'n',
                  'hun': 'y',
                  'pat': 'full',
                  'price': '$',
                  'rain': 'n',
                  'res': 'n',
                  'type': 'thai',
                  'est': 'b3060'},
                 False)

    e3 = Example({'alt': 'n',
                  'bar': 'y',
                  'fri': 'n',
                  'hun': 'n',
                  'pat': 'some',
                  'price': '$',
                  'rain': 'n',
                  'res': 'n',
                  'type': 'burger',
                  'est': 'b010'},
                 True)

    e4 = Example({'alt': 'y',
                  'bar': 'n',
                  'fri': 'y',
                  'hun': 'y',
                  'pat': 'full',
                  'price': '$',
                  'rain': 'y',
                  'res': 'n',
                  'type': 'thai',
                  'est': 'b1030'},
                 True)

    e5 = Example({'alt': 'y',
                  'bar': 'n',
                  'fri': 'y',
                  'hun': 'n',
                  'pat': 'full',
                  'price': '$$$',
                  'rain': 'n',
                  'res': 'y',
                  'type': 'french',
                  'est': 'g60'},
                 False)

    e6 = Example({'alt': 'n',
                  'bar': 'y',
                  'fri': 'n',
                  'hun': 'y',
                  'pat': 'some',
                  'price': '$$',
                  'rain': 'y',
                  'res': 'y',
                  'type': 'italian',
                  'est': 'b010'},
                 True)

    e7 = Example({'alt': 'n',
                  'bar': 'y',
                  'fri': 'n',
                  'hun': 'n',
                  'pat': 'none',
                  'price': '$',
                  'rain': 'y',
                  'res': 'n',
                  'type': 'burger',
                  'est': 'b010'},
                 False)

    e8 = Example({'alt': 'n',
                  'bar': 'n',
                  'fri': 'n',
                  'hun': 'y',
                  'pat': 'some',
                  'price': '$$',
                  'rain': 'y',
                  'res': 'y',
                  'type': 'thai',
                  'est': 'b010'},
                 True)

    e9 = Example({'alt': 'n',
                  'bar': 'y',
                  'fri': 'y',
                  'hun': 'n',
                  'pat': 'full',
                  'price': '$',
                  'rain': 'y',
                  'res': 'n',
                  'type': 'burger',
                  'est': 'g60'},
                 False)

    e10 = Example({'alt': 'y',
                  'bar': 'y',
                  'fri': 'y',
                  'hun': 'y',
                  'pat': 'full',
                  'price': '$$$',
                  'rain': 'n',
                  'res': 'y',
                  'type': 'italian',
                  'est': 'b1030'},
                 False)

    e11 = Example({'alt': 'n',
                  'bar': 'n',
                  'fri': 'n',
                  'hun': 'n',
                  'pat': 'none',
                  'price': '$',
                  'rain': 'n',
                  'res': 'n',
                  'type': 'thai',
                  'est': 'b010'},
                 False)

    e12 = Example({'alt': 'y',
                  'bar': 'y',
                  'fri': 'y',
                  'hun': 'y',
                  'pat': 'full',
                  'price': '$',
                  'rain': 'n',
                  'res': 'n',
                  'type': 'burger',
                  'est': 'b3060'},
                 True)

    examples = ExampleList([e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12])

    runTest(attrSpecs, classAttrSpec, defaultClass, examples)

#########################################################################

def mTest():
    '''
    This example is from Tom Mitchell's "Machine Learning" text.
    "Is today a good day to play tennis?"
    '''

    # Define the attribute specifications.
    attrSpecs = [AttributeSpec("Outlook", ['Sunny', 'Overcast', 'Rain']),
                 AttributeSpec("Temperature", ['Hot', 'Mild', 'Cool']),
                 AttributeSpec("Humidity", ['High', 'Normal']),
                 AttributeSpec("Wind", ['Strong', 'Weak'])]
    classAttrSpec = AttributeSpec("Play tennis?", ['N', 'Y'])
    defaultClass = 'Y'

    # Define the examples.
    e1 = Example({'Outlook': 'Sunny',
                  'Temperature': 'Hot',
                  'Humidity': 'High',
                  'Wind': 'Weak'},
                 'N')

    e2 = Example({'Outlook': 'Sunny',
                  'Temperature': 'Hot',
                  'Humidity': 'High',
                  'Wind': 'Strong'},
                 'N')

    e3 = Example({'Outlook': 'Overcast',
                  'Temperature': 'Hot',
                  'Humidity': 'High',
                  'Wind': 'Weak'},
                 'Y')

    e4 = Example({'Outlook': 'Rain',
                  'Temperature': 'Mild',
                  'Humidity': 'High',
                  'Wind': 'Weak'},
                 'Y')

    e5 = Example({'Outlook': 'Rain',
                  'Temperature': 'Cool',
                  'Humidity': 'Normal',
                  'Wind': 'Weak'},
                 'Y')

    e6 = Example({'Outlook': 'Rain',
                  'Temperature': 'Cool',
                  'Humidity': 'Normal',
                  'Wind': 'Strong'},
                 'N')

    e7 = Example({'Outlook': 'Overcast',
                  'Temperature': 'Cool',
                  'Humidity': 'Normal',
                  'Wind': 'Strong'},
                 'Y')

    e8 = Example({'Outlook': 'Sunny',
                  'Temperature': 'Mild',
                  'Humidity': 'High',
                  'Wind': 'Weak'},
                 'N')

    e9 = Example({'Outlook': 'Sunny',
                  'Temperature': 'Cool',
                  'Humidity': 'Normal',
                  'Wind': 'Weak'},
                 'Y')

    e10 = Example({'Outlook': 'Rain',
                  'Temperature': 'Mild',
                  'Humidity': 'Normal',
                  'Wind': 'Weak'},
                 'Y')

    e11 = Example({'Outlook': 'Sunny',
                  'Temperature': 'Mild',
                  'Humidity': 'Normal',
                  'Wind': 'Strong'},
                 'Y')

    e12 = Example({'Outlook': 'Overcast',
                  'Temperature': 'Mild',
                  'Humidity': 'High',
                  'Wind': 'Strong'},
                 'Y')

    e13 = Example({'Outlook': 'Overcast',
                  'Temperature': 'Hot',
                  'Humidity': 'Normal',
                  'Wind': 'Weak'},
                 'Y')

    e14 = Example({'Outlook': 'Rain',
                  'Temperature': 'Mild',
                  'Humidity': 'High',
                  'Wind': 'Strong'},
                 'N')

    examples = ExampleList([e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14])

    runTest(attrSpecs, classAttrSpec, defaultClass, examples)


from lxml import html
import requests
from directoryref import HTMLReportDirectory

page = requests.get("http://www.nhl.com/ice/teams.htm")
tree = html.fromstring(page.text)

TeamPlaceList = tree.xpath('//span[@class="teamPlace"]/text()')
TeamCommonList = tree.xpath('//span[@class="teamCommon"]/text()')

TeamTuple = list(set(zip(TeamPlaceList, TeamCommonList)))
TeamList = []
for teams in TeamTuple:
	TeamList = [' '.join(teams) for teams in TeamTuple]

rawSrc = HTMLReportDirectory + "\\List of NHL Teams.txt"
wRawSrc = open(rawSrc, "w")

for team in sorted(TeamList):
	printinfo = "%s\n" % team
	wRawSrc.write(printinfo.encode('utf8'))


import os
import time

abooklist=list()

menu = """
 Welcome to the Library
 1-List of Books
 2-Donate a Book to Library
 3-Take a Book from Library
 4-Exit
 """

def listofbooks(booklist:list):
    num=len(booklist)
    if num==0:
        print("There is no book in the library.\n Please donate a book.")
        print("Please press Enter to return main menu")
        input()
    else:
        counter=0
        for i in booklist:
            counter+=1
            print(counter,".Book=" ,i)
        print("Please press Enter to return main menu")
        input()

def donatebook(bookname:str,booklist:list):
    booklist.append(bookname)
    print("The book you entered is adding to the library")
    time.sleep(1)
    print("-\-")
    time.sleep(1)
    print("-/-")
    time.sleep(1)
    print("The book you entered has been added to library")
    print("Please press Enter to return main menu")
    input()

def takebook(bookname:str,booklist:list):
    counter=0
    for i in booklist:
        if i==bookname:
            counter+=1
    if counter>0:
        booklist.remove(bookname)
        print("The book you entered is removing from library")
        time.sleep(1)
        print("-\-")
        time.sleep(1)
        print("-/-")
        time.sleep(1)
        print("The book you entered has been removed from library")
        print("Please press Enter to return main menu")
        input()
    else:
        print("The book you entered was not found!")
        print("Please press Enter to return main menu")
        input()
while True:
    print("\n"*100)
    print(menu)
    choice=input("Your choice=")
    if choice=="1":
        listofbooks(abooklist)
    if choice=="2":
        book=input("Please write book's name which is donating")
        donatebook(book,abooklist)
    if choice=="3":
        book=input("Please write book's name that you want")
        takebook(book,abooklist)
    if choice=="4":
        print("Exitting")
        time.sleep(1)
        print("Exited!\nHave a good day")
        break
    else:
        print("You entered a wrong value, please try again.")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jan 13 17:47:04 2018

@author: barbara
"""

#namepath = 'bgris'
namepath = 'barbara'

import sys
sys.path.insert(0, '/home/' + namepath)
import odl
import numpy as np

##%% Create data from lddmm registration
import matplotlib.pyplot as plt

from DeformationModulesODL.deform import Kernel
from DeformationModulesODL.deform import DeformationModuleAbstract
from DeformationModulesODL.deform import SumTranslations
from DeformationModulesODL.deform import UnconstrainedAffine
from DeformationModulesODL.deform import LocalScaling
from DeformationModulesODL.deform import LocalRotation
from DeformationModulesODL.deform import EllipseMvt
from DeformationModulesODL.deform import FromFile
from DeformationModulesODL.deform import FromFileV5
from DeformationModulesODL.deform import TemporalAttachmentModulesGeom
from DeformationModulesODL.deform import FromPointsVectorsCoeff_MultiDim
import odl
import scipy.ndimage as ndimage

import numpy as np
from DeformationModulesODL.deform import FromPointsVectorsCoeff

import charestimate as char
import estimate_structured_base_pointsvectors as est_coeff
import function_compute_pointsvectors as cmp
#import generate_data_doigt as gen
import structured_vector_fields as struct
import function_generate_data_doigt_bis as fun_gen

import scipy

# Discrete reconstruction space: discretized functions on the rectangle
space = odl.uniform_discr(
    min_pt=[-10, -10], max_pt=[10, 10], shape=[512,512],
    dtype='float32', interp='linear')

width = 2

r_b = 4
r_c = 4
sigma = 0.2

nbdata = 10
mg = space.meshgrid
dim = 2

#%% Figure data
def kernel_np(x, y):
    #si = tf.shape(x)[0]
    return np.exp(- sum([ (x[i] - y[i]) ** 2 for i in range(dim)]) / (sigma ** 2))

sblur = 5
namepath = 'bgris'
namepath = 'barbara'
path = '/home/' + namepath + '/data/'
pathresult = '/home/' + namepath + '/Results/DeformationModules/'
pathexp = 'Doigtbis_dimcont2/'
#pathexp = 'RotationTranslationRectangle_dimcont2/'

path += pathexp
#path = '/home/bgris/data/Doigtbis/'
#name_exp = 'rb_' + str(r_b) +  '_width_' + str(width) + '_sigma_' + str(sigma) + '_nbdata_' + str(nbdata)
name_exp = 'rb_' + str(r_b) + '_rc_' + str(r_c) + '_width_' + str(width) + '_sigma_' + str(sigma) + '_nbdata_' + str(nbdata) + '_sblur_' + str(sblur)
#name_exp = 'rb_' + str(r_b) + '_width_' + str(width) + '_sigma_' + str(sigma) + 'nb_fixed' + '_nbdata_' + str(nbdata)
#name_exp = 'rb_' + str(r_b) + '_rc_' + str(r_c) + '_width_' + str(width) + '_sigma_' + str(sigma) + '_nbdata_' + str(nbdata)

name = path + name_exp + '/'

structured_list = []
unstructured_list = []
points_list = []
vectors_list = []
cov_mat_list = []
param_list = []
A_inner_prod_list = []
image_list = []
nbdatamax = 5
for i in range(nbdatamax):
    structured_list.append(np.loadtxt(name + 'structured' + str(i)))
    unstructured_list.append(np.loadtxt(name + 'unstructured' + str(i)))
    vectors_i = structured_list[i][dim:2*dim]
    points_list.append(np.loadtxt(name + 'points' + str(i)))
    vectors_list.append(np.loadtxt(name + 'vectors' + str(i)))
    param_tmp = np.loadtxt(name + 'param' + str(i))
    param_list.append(param_tmp)
    image_temp = fun_gen.generate_image_2articulations(space, param_tmp[0:2], param_tmp[2:4],param_tmp[2:4] + 0.85*( param_tmp[4:6] - param_tmp[2:4]), 0.5*width).copy()
    image_list.append(space.element(scipy.ndimage.filters.gaussian_filter(image_temp, 10)))
    cov_mat_list.append(struct.make_covariance_matrix(points_list[i], kernel_np))
    A_inner_prod_list.append(np.dot(cov_mat_list[i], np.dot(vectors_i.T, vectors_list[i] )).T)
#

param_list = np.array(param_list).T

nb_points = len(points_list[0][0])
nb_vectors = len(vectors_list[0][0])

points = space.points()
step = 40
fac= 0.1
import os
namefig_init = pathresult + pathexp + name_exp

#%% Data
#gen_unstructured = struct.get_from_structured_to_unstructured(space, kernel_np)

#os.mkdir(namefig_init)

# save images
for i in range(nbdatamax):
    namefig = namefig_init + '/' + 'data_image_' + str(i)
    fig = image_list[i].show(clim=[0,1])
    plt.axis('off')
    fig.delaxes(fig.axes[1])
    plt.savefig(namefig)

#

# save images + vector fields
for i in range(nbdatamax):
    namefig = namefig_init + '/' + 'data_image_vectfield' + str(i)
    fig = image_list[i].show(clim=[0,1])
    plt.axis('off')
    fig.delaxes(fig.axes[1])
    v = unstructured_list[i].copy()
    #v = gen_unstructured(structured_list[i])
    plt.quiver(points.T[0][::step],points.T[1][::step],fac*v[0][::step],fac*v[1][::step], color='r')

    plt.savefig(namefig)

#

# save images + GD
for i in range(nbdatamax):
    namefig = namefig_init + '/' + 'data_image_GD' + str(i)
    fig = image_list[i].show(clim=[0,1])
    plt.axis('off')
    fig.delaxes(fig.axes[1])
    plt.plot(param_list.T[i][::2], param_list.T[i][1::2],'xb')
    plt.savefig(namefig)

#

# save images + vector fields + GD
for i in range(nbdatamax):
    namefig = namefig_init + '/' + 'data_image_vectfield_GD' + str(i)
    fig = image_list[i].show(clim=[0,1])
    plt.axis('off')
    fig.delaxes(fig.axes[1])
    v = unstructured_list[i].copy()
    #v = gen_unstructured(structured_list[i])
    plt.quiver(points.T[0][::step],points.T[1][::step],fac*v[0][::step],fac*v[1][::step], color='r')

    plt.plot(param_list.T[i][::2], param_list.T[i][1::2],'xb')
    plt.savefig(namefig)

#

#%% Rsource and target

image = space.element(np.loadtxt('/home/barbara/data/Doigtbis_dimcont2/ex_source/11'))
image = space.element(scipy.ndimage.filters.gaussian_filter(image, 10))
namefig = '/home/barbara/data/Doigtbis_dimcont2/ex_source/11.png'
fig = image.show(clim=[0,1])
plt.axis('off')
fig.delaxes(fig.axes[1])
plt.savefig(namefig)


#     # window size is WxW
# C_Thr = 0.43    # threshold for coherency
# LowThr = 35     # threshold1 for orientation, it ranges from 0 to 180
# HighThr = 57    # threshold2 for orientation, it ranges from 0 to 180

# from

import numpy as np
import cv2 as cv

def calcGST(inputIMG, w):
    """
    from https://docs.opencv.org/4.2.0/d4/d70/tutorial_anisotropic_image_segmentation_by_a_gst.html
    :param inputIMG:
    :param w:
    :return:
    """
    img = inputIMG.astype(np.float32)
    # GST components calculation (start)
    # J =  (J11 J12; J12 J22) - GST
    img_diff_x = cv.Sobel(img, cv.CV_32F, 1, 0, 3) #Calculates the first image derivatives using an extended Sobel operator
    img_diff_y = cv.Sobel(img, cv.CV_32F, 0, 1, 3)
    img_diff_xy = cv.multiply(img_diff_x, img_diff_y)

    img_diff_xx = cv.multiply(img_diff_x, img_diff_x)
    img_diff_yy = cv.multiply(img_diff_y, img_diff_y)
    J11 = cv.boxFilter(img_diff_xx, cv.CV_32F, (w, w))
    J22 = cv.boxFilter(img_diff_yy, cv.CV_32F, (w, w))
    J12 = cv.boxFilter(img_diff_xy, cv.CV_32F, (w, w))
    # GST components calculations (stop)
    # eigenvalue calculation (start)
    # lambda1 = J11 + J22 + sqrt((J11-J22)^2 + 4*J12^2)
    # lambda2 = J11 + J22 - sqrt((J11-J22)^2 + 4*J12^2)
    tmp1 = J11 + J22
    tmp2 = J11 - J22
    tmp2 = cv.multiply(tmp2, tmp2)
    tmp3 = cv.multiply(J12, J12)
    tmp4 = np.sqrt(tmp2 + 4.0 * tmp3)
    lambda1 = tmp1 + tmp4  # biggest eigenvalue
    lambda2 = tmp1 - tmp4  # smallest eigenvalue
    # eigenvalue calculation (stop)
    # Coherency calculation (start)
    # Coherency = (lambda1 - lambda2)/(lambda1 + lambda2)) - measure of anisotropism
    # Coherency is anisotropy degree (consistency of local orientation)
    coherency = cv.divide(lambda1 - lambda2, lambda1 + lambda2)
    # Coherency calculation (stop)
    # orientation angle calculation (start)
    # tan(2*Alpha) = 2*J12/(J22 - J11)
    # Alpha = 0.5 atan2(2*J12/(J22 - J11))
    orientation = cv.phase(J22 - J11, 2.0 * J12, angleInDegrees=True)
    orientation = 0.5 * orientation
    # orientation angle calculation (stop)
    return coherency, orientation

def ApplyDenoising(img_pathname, filterstrenght=20):  #es: "sharad_data/s_00429402_thm.jpg" or "sharad_data/s_00387302_thm.jpg"
    imgIn_pre = cv.imread(img_pathname, cv.IMREAD_GRAYSCALE)

    imgIn = cv.fastNlMeansDenoising(imgIn_pre,None,filterstrenght,7,21)
    return imgIn

def ApplyStructureTensor(image, xin=2600, yin=1700, xstep=2400, ystep=1700, W = 10):
    #imgInSquare = imgIn[1600:3600,2900:4900]
    #imgInTiny = imgIn[1700:3400,2600:5000]
    imgInTiny = image[yin:yin+ystep,xin:xin+xstep]

    imgCoherency, ori = calcGST(imgInTiny, W)
    #imgCoherencySquare, oriSquare = calcGST(imgInSquare, W)

    ori[ori>=90] -=180

    _, imgCoherencyBin = cv.threshold(imgCoherency,0.1, 255, cv.THRESH_BINARY) #2nd value: C_Thr
    #_, imgCoherencyBinSquare = cv.threshold(imgCoherencySquare,0.1, 255, cv.THRESH_BINARY)

    ori_bin = cv.inRange(ori, -50, 50)
    #ori_binSquare = cv.inRange(oriSquare, -50, 50)

    imgBin = cv.bitwise_and(imgCoherencyBin, ori_bin.astype(np.float32))
    #imgBinSquare = cv.bitwise_and(imgCoherencyBinSquare, ori_binSquare.astype(np.float32))
    dilation_size = 5
    element = cv.getStructuringElement( cv.MORPH_ELLIPSE,
                                       ( 2*dilation_size + 1, 2*dilation_size+1 ),
                                       ( dilation_size, dilation_size ) );
    eroded = cv.erode(imgBin, element)
    imgBinClear = cv.dilate(eroded, element)

    return imgInTiny, imgCoherency, ori, imgCoherencyBin, ori_bin, imgBin, imgBinClear


import jongau.identity
import jongau.settings
import requests
import logging
from rdflib import BNode, Graph, Literal, Namespace, URIRef
from rdflib.namespace import RDF, RDFS, FOAF, XSD
CERT = Namespace('http://www.w3.org/ns/auth/cert#')

def fetch_webid(webid):
	""" Fetches an rdf object without sending a webid
	    A webid rdf object should not require authentication
	    Returns an rdflib graph, which will have content unless there was an error
	"""
	graph = Graph('IOMemory', BNode())
	headers = {'Content-type': 'text/turtle'}
	try:
		request = requests.get(webid, headers=headers)
		graph.parse(data=request.text, format='turtle')
	except Exception as e:
		logging.warning("Failed to fetch webid %s: %s"%(webid, e))
	query = 'SELECT ?source WHERE { <%s> rdf:sameAs ?source . }' % (webid,)
	res = graph.query(query)
	for source in res:
		parsed_source = fetch_rdf(source[0])
		if parsed_source:
			graph = graph + parsed_source
	return graph

def fetch_rdf(rdf, cert=None):
	""" Fetches an rdf object and sends the server's webid for authentication
	    A custom (cert, key) pair can be given, instead of using the server's webid
	    Returns an rdflib graph, which will have content unless there was an error
	"""
	graph = Graph('IOMemory', BNode())
	headers = {'Content-type': 'text/turtle'}
	if not cert:
		cert = jongau.identity.get_client_cert()
	try:
		request = requests.get(rdf, headers=headers, cert=cert)
		graph.parse(data=request.text, format='turtle')
	except Exception as e:
		logging.warning("Failed to fetch rdf url %s: %s"%(rdf, e))
	return graph

def get_server_root(webid):
	index = webid.find('/webid/')
	if index >= 0:
		return webid[:index]
	return webid


import numpy as np

import pandas as pd
from pandas.plotting import scatter_matrix

from scipy import stats

import matplotlib.pyplot as plt
import seaborn as sns

from random import randrange

import re

import random
import pickle,pprint

from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter

U_t=15    # mV
D=2       # mV
C=1   # 10Hz-> 0.01ms^{-1}
dt=1    #ms
tau=20    #ms

def mu(t):
    return 16

t_window = 100
dt = 0.1
N = int(t_window / dt)

ur = np.zeros((N + 1, 1))
rho = np.zeros((N + 1, 1))

mu_ = 16

for r in range(N):
    ur[r + 1] = ur[r] * (1 - dt / tau) + mu_ * dt / tau

for r in range(N + 1):
    rho[r] = C * np.exp((ur[r] - U_t) / D)

int_rho = np.zeros((N + 1, 1))

for r in range(N):
    int_rho[r + 1] = int_rho[r] + (rho[r] + rho[r + 1]) / 2 * dt

exp_rho = np.exp(-int_rho[1:])

def int_tau(l,exp_rho):
    sum_=0
    for r in range(N):
        sum_+=np.exp(-l*dt*(r+0.5))*exp_rho[r]*(rho[r]+rho[r+1])/2*dt
    return abs(sum_)

Grid_size=500

real_array=np.linspace(-1.5,0.,Grid_size)
im_array=np.linspace(0,3,Grid_size)

Grid=np.zeros((Grid_size,Grid_size))
for re in range(Grid_size):
    print(re)
    for im in range(Grid_size):
        Grid[re,im]=int_tau(real_array[re]+im_array[im]*1j,exp_rho)

Grid_L = {'Grid': Grid}

output = open('Grid_laplace.pkl', 'wb')

# Pickle dictionary using protocol 0.
pickle.dump(Grid_L, output)

output.close()


import socket
import struct
import os
import hashlib

ip_address='127.0.0.1'
port_number=3333

server_sock=socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_sock.bind((ip_address,port_number))
print("server socket open...")

##########
recv_num=0

total_size=0

while 1:
	file_info,addr=server_sock.recvfrom(1045)
	checksum=file_info[:20]
	sequence_num=file_info[20:21]
	data=file_info[21:]

	if sequence_num.decode() == str(recv_num%2):
		h=hashlib.sha1()
		h.update(sequence_num+data)
		check_recv=h.digest()
		file_path=""

		if check_recv == checksum:
			recv_num+=1
			total_size=struct.unpack("!i",data[:4])[0]
			file_name=data[4:].decode()

			file_path="./received_dir/"+file_name
			new_file=open(file_path,"wb")
			print("Send file info ACK..")
			ack_str="ACK"+str(recv_num%2)
			server_sock.sendto(ack_str.encode(), addr)
			break
		else:
			recv_num=0
			print("Send file info ACK..")
			ack_str="ACK"+str(recv_num%2)
			server_sock.sendto(ack_str.encode(), addr)

file2=open(file_path,"wb")
print("file Name = "+file_name)
print("file Size = "+str(total_size))
print("received file Path = "+file_path)
current_size=0

while current_size != total_size:

	file_data,addr=server_sock.recvfrom(1045)
	checksum=file_data[:20]
	sequence_num=file_data[20:21]
	data2=file_data[21:]

	if sequence_num.decode() == str(recv_num%2):
		h=hashlib.sha1()
		h.update(sequence_num+data2)

		check_recv=h.digest()

		if check_recv == checksum:
			recv_num+=1

			if total_size-current_size >= 1024:
				current_size+=1024

				file2.write(data2)
			else:

				current_size+=total_size-current_size
				file2.write(data2)

			progress=(current_size/total_size)*100
			print("(currenet size / total size) = "+str(current_size)+"/"+str(total_size)+" , "+str(round(progress,3))+" % ")

			ack_str="ACK"+str(recv_num%2)
			server_sock.sendto(ack_str.encode(), addr)

		else:
			recv_num-=1
			ack_str="ACK"+str(recv_num%2)
			server_sock.sendto(ack_str.encode(), addr)

file2.close()
print("File Receive End.")