# coding: utf-8

"""
    LoRa App Server REST API

     For more information about the usage of the LoRa App Server (REST) API, see [https://docs.loraserver.io/lora-app-server/api/](https://docs.loraserver.io/lora-app-server/api/).   # noqa: E501

    OpenAPI spec version: 1.0.0

    Generated by: https://github.com/swagger-api/swagger-codegen.git
"""

import pprint
import re  # noqa: F401

import six

class ApiApplication(object):
    """NOTE: This class is auto generated by the swagger code generator program.

    Do not edit the class manually.
    """

    """
    Attributes:
      swagger_types (dict): The key is attribute name
                            and the value is attribute type.
      attribute_map (dict): The key is attribute name
                            and the value is json key in definition.
    """
    swagger_types = {
        'description': 'str',
        'id': 'str',
        'name': 'str',
        'organization_id': 'str',
        'payload_codec': 'str',
        'payload_decoder_script': 'str',
        'payload_encoder_script': 'str',
        'service_profile_id': 'str'
    }

    attribute_map = {
        'description': 'description',
        'id': 'id',
        'name': 'name',
        'organization_id': 'organizationID',
        'payload_codec': 'payloadCodec',
        'payload_decoder_script': 'payloadDecoderScript',
        'payload_encoder_script': 'payloadEncoderScript',
        'service_profile_id': 'serviceProfileID'
    }

    def __init__(self, description=None, id=None, name=None, organization_id=None, payload_codec=None, payload_decoder_script=None, payload_encoder_script=None, service_profile_id=None):  # noqa: E501
        """ApiApplication - a model defined in Swagger"""  # noqa: E501

        self._description = None
        self._id = None
        self._name = None
        self._organization_id = None
        self._payload_codec = None
        self._payload_decoder_script = None
        self._payload_encoder_script = None
        self._service_profile_id = None
        self.discriminator = None

        if description is not None:
            self.description = description
        if id is not None:
            self.id = id
        if name is not None:
            self.name = name
        if organization_id is not None:
            self.organization_id = organization_id
        if payload_codec is not None:
            self.payload_codec = payload_codec
        if payload_decoder_script is not None:
            self.payload_decoder_script = payload_decoder_script
        if payload_encoder_script is not None:
            self.payload_encoder_script = payload_encoder_script
        if service_profile_id is not None:
            self.service_profile_id = service_profile_id

    @property
    def description(self):
        """Gets the description of this ApiApplication.  # noqa: E501

        Description of the application.  # noqa: E501

        :return: The description of this ApiApplication.  # noqa: E501
        :rtype: str
        """
        return self._description

    @description.setter
    def description(self, description):
        """Sets the description of this ApiApplication.

        Description of the application.  # noqa: E501

        :param description: The description of this ApiApplication.  # noqa: E501
        :type: str
        """

        self._description = description

    @property
    def id(self):
        """Gets the id of this ApiApplication.  # noqa: E501

        Application ID. This will be automatically assigned on create.  # noqa: E501

        :return: The id of this ApiApplication.  # noqa: E501
        :rtype: str
        """
        return self._id

    @id.setter
    def id(self, id):
        """Sets the id of this ApiApplication.

        Application ID. This will be automatically assigned on create.  # noqa: E501

        :param id: The id of this ApiApplication.  # noqa: E501
        :type: str
        """

        self._id = id

    @property
    def name(self):
        """Gets the name of this ApiApplication.  # noqa: E501

        Name of the application (must be unique).  # noqa: E501

        :return: The name of this ApiApplication.  # noqa: E501
        :rtype: str
        """
        return self._name

    @name.setter
    def name(self, name):
        """Sets the name of this ApiApplication.

        Name of the application (must be unique).  # noqa: E501

        :param name: The name of this ApiApplication.  # noqa: E501
        :type: str
        """

        self._name = name

    @property
    def organization_id(self):
        """Gets the organization_id of this ApiApplication.  # noqa: E501

        ID of the organization to which the application belongs. After create, this can not be modified.  # noqa: E501

        :return: The organization_id of this ApiApplication.  # noqa: E501
        :rtype: str
        """
        return self._organization_id

    @organization_id.setter
    def organization_id(self, organization_id):
        """Sets the organization_id of this ApiApplication.

        ID of the organization to which the application belongs. After create, this can not be modified.  # noqa: E501

        :param organization_id: The organization_id of this ApiApplication.  # noqa: E501
        :type: str
        """

        self._organization_id = organization_id

    @property
    def payload_codec(self):
        """Gets the payload_codec of this ApiApplication.  # noqa: E501

        Payload codec. NOTE: These field have moved to the device-profile and will be removed in the next major release. When set, the device-profile payload_ fields have priority over the application payload_ fields.  # noqa: E501

        :return: The payload_codec of this ApiApplication.  # noqa: E501
        :rtype: str
        """
        return self._payload_codec

    @payload_codec.setter
    def payload_codec(self, payload_codec):
        """Sets the payload_codec of this ApiApplication.

        Payload codec. NOTE: These field have moved to the device-profile and will be removed in the next major release. When set, the device-profile payload_ fields have priority over the application payload_ fields.  # noqa: E501

        :param payload_codec: The payload_codec of this ApiApplication.  # noqa: E501
        :type: str
        """

        self._payload_codec = payload_codec

    @property
    def payload_decoder_script(self):
        """Gets the payload_decoder_script of this ApiApplication.  # noqa: E501

        Payload decoder script. NOTE: These field have moved to the device-profile and will be removed in the next major release. When set, the device-profile payload_ fields have priority over the application payload_ fields.  # noqa: E501

        :return: The payload_decoder_script of this ApiApplication.  # noqa: E501
        :rtype: str
        """
        return self._payload_decoder_script

    @payload_decoder_script.setter
    def payload_decoder_script(self, payload_decoder_script):
        """Sets the payload_decoder_script of this ApiApplication.

        Payload decoder script. NOTE: These field have moved to the device-profile and will be removed in the next major release. When set, the device-profile payload_ fields have priority over the application payload_ fields.  # noqa: E501

        :param payload_decoder_script: The payload_decoder_script of this ApiApplication.  # noqa: E501
        :type: str
        """

        self._payload_decoder_script = payload_decoder_script

    @property
    def payload_encoder_script(self):
        """Gets the payload_encoder_script of this ApiApplication.  # noqa: E501

        Payload encoder script. NOTE: These field have moved to the device-profile and will be removed in the next major release. When set, the device-profile payload_ fields have priority over the application payload_ fields.  # noqa: E501

        :return: The payload_encoder_script of this ApiApplication.  # noqa: E501
        :rtype: str
        """
        return self._payload_encoder_script

    @payload_encoder_script.setter
    def payload_encoder_script(self, payload_encoder_script):
        """Sets the payload_encoder_script of this ApiApplication.

        Payload encoder script. NOTE: These field have moved to the device-profile and will be removed in the next major release. When set, the device-profile payload_ fields have priority over the application payload_ fields.  # noqa: E501

        :param payload_encoder_script: The payload_encoder_script of this ApiApplication.  # noqa: E501
        :type: str
        """

        self._payload_encoder_script = payload_encoder_script

    @property
    def service_profile_id(self):
        """Gets the service_profile_id of this ApiApplication.  # noqa: E501

        ID of the service profile.  # noqa: E501

        :return: The service_profile_id of this ApiApplication.  # noqa: E501
        :rtype: str
        """
        return self._service_profile_id

    @service_profile_id.setter
    def service_profile_id(self, service_profile_id):
        """Sets the service_profile_id of this ApiApplication.

        ID of the service profile.  # noqa: E501

        :param service_profile_id: The service_profile_id of this ApiApplication.  # noqa: E501
        :type: str
        """

        self._service_profile_id = service_profile_id

    def to_dict(self):
        """Returns the model properties as a dict"""
        result = {}

        for attr, _ in six.iteritems(self.swagger_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value
        if issubclass(ApiApplication, dict):
            for key, value in self.items():
                result[key] = value

        return result

    def to_str(self):
        """Returns the string representation of the model"""
        return pprint.pformat(self.to_dict())

    def __repr__(self):
        """For `print` and `pprint`"""
        return self.to_str()

    def __eq__(self, other):
        """Returns true if both objects are equal"""
        if not isinstance(other, ApiApplication):
            return False

        return self.__dict__ == other.__dict__

    def __ne__(self, other):
        """Returns true if both objects are not equal"""
        return not self == other


#!/usr/bin/env python
# coding: utf-8

# посмотреть ошибки mail
# open /var/mail

# crontab -e
# 0 9-18 * * * /Users/molchanov/opt/anaconda3/bin/python3 /Users/molchanov/projects/parser/solid/euro_currency.py
# https://scrapfly.io/blog/scraping-using-browsers/

import os
import requests
from bs4 import BeautifulSoup

# библиотека для динамических web
from selenium import webdriver

import warnings
warnings.filterwarnings("ignore")

# через cronetab не видет путь
cwd = '/Users/molchanov/dev/solid_cur' # os.getcwd()

with open(f'{cwd}/url.txt') as f:
    main_url = f.read()

def get_bot_info(file_path):
    # Open the file in read mode
    with open(file_path, "r") as file:
        # Read the entire contents of the file
        file_contents = file.readlines()

    return [f.strip() for f in file_contents]

def telegram_bot_sendtext(bot_message):

    bot_info = get_bot_info(f'{cwd}/bot_info.txt')
    bot_token = bot_info[0]
    bot_chatID = bot_info[1]
    send_text = ('https://api.telegram.org/bot' +
                bot_token + '/sendMessage?chat_id=' +
                bot_chatID + '&parse_mode=Markdown&text=' + bot_message)

    response = requests.get(send_text)

    return response.json()

def main():
    options = webdriver.ChromeOptions()
    options.add_argument("headless")
    driver = webdriver.Chrome(executable_path='./chromedriver', chrome_options=options)
    driver.get(main_url)

    html = driver.page_source
    soup = BeautifulSoup(html, "html.parser")
    currency = soup.find_all('tbody', id='currency_rate_solid')

    _, dol_2, dol_3, _, eur_2, eur_3 = currency[0].text.split(',')[:6]

    usd = float('.'.join([dol_2[2:], dol_3[:2]]))
    eur = float('.'.join([eur_2[2:], eur_3[:2]]))

    # telegram_bot_sendtext('check')

    with open(f'{cwd}/cur_eur.txt') as f:
        cur_eur = f.read()

    if eur != float(cur_eur):

        telegram_bot_sendtext(
            f"usd: {str(usd)} \
| eur: {str(eur)} \
| {round(eur/usd, 4)}")

        with open(f'{cwd}/cur_eur.txt', 'w') as f:
            f.write(str(eur))

if __name__ == '__main__':
    main()


import numpy as np
import pickle
import os
import sys
sys.path.append(
    '/Users/lindawu/Desktop/OneDrive/School Work/HCI Lab/2019 social robot/codes/interaction-adaptation-working')

def data_preprocessor (input_style):
    # inputs from experiments
    if input_style == 1:
        # import and combine data files
        with open('learner/history.pkl', 'rb') as f:
            raw_data = pickle.load(f, encoding='bytes')

        # Read every pickle data file and combine data
        paths = ["learner/linda1", "learner/linda2"]
        for path in paths:
            for folderName in os.listdir(path):
                folderPath = path + "/" + folderName
                for file in os.listdir(folderPath):
                    if ".pkl" in file:
                        filename = file
                filePath = folderPath + "/" + filename
                with open(filePath, 'rb') as f:
                    if "union" in filePath:
                        location = "union"  # TODO: use the location for later maybe
                    else:
                        location = "DOIT"
                    temp = pickle.load(f, encoding='bytes')
                    raw_data.extend(temp)

        # for r in raw_data:
        #     print("\n\n{}".format(r))

        # parse our raw data and keep useful vectors
        trajs = []
        time_stamps = []

        for raw_traj in raw_data:
            end_time = raw_traj.pop(-1)  # end time
            start_time = raw_traj.pop(-1)  # start time
            date = raw_traj.pop(-1)  # date
            time_stamp = [end_time, start_time, date]
            if time_stamp in time_stamps:
                continue
            time_stamps.append(time_stamp)

            _ = raw_traj.pop(-1)  # id
            _ = raw_traj.pop(-1)  # mutated?

            is_correctness = raw_traj.pop(-1)
            is_prefix = raw_traj.pop(-1)
            score = raw_traj.pop(-1)

            # loop through all entries, get trajectories and human status
            human_input = "Ready"
            micro = raw_traj[0][0]
            traj_vect = [(human_input, micro)]
            i = 1
            while i < len(raw_traj) - 1:

                micro = raw_traj[i + 1][0]
                raw_human_input = raw_traj[i][-1]
                if raw_human_input.decode("utf-8") == "human_ready":
                    human_input = "Ready"
                else:
                    human_input = "Ignore"

                item = (human_input, micro)
                traj_vect.append(item)

                i += 2
            # the last human status
            micro = 'EndInteraction'
            raw_human_input = raw_traj[len(raw_traj) - 1][-1]
            if raw_human_input.decode("utf-8") == "human_ready":
                human_input = "Ready"
            else:
                human_input = "Ignore"
            item = (human_input, micro)
            traj_vect.append(item)

            # append each traj item to list of trajs
            traj = {'traj_vect': traj_vect, 'score': score, 'is_prefix':
                is_prefix, 'is_correctness': is_correctness}
            trajs.append(traj)

    # inputs from simulated data
    if input_style == 2:
        with open('inputs/generated_data/history.pkl', 'rb') as f:
            raw_data = pickle.load(f, encoding='bytes')

        trajs = []
        for r in raw_data:
            traj_vect = []
            vect = r.vect
            for v in vect:
                human_input = v[0].type
                micro = v[1].type
                item = (human_input, micro)
                traj_vect.append(item)
            score = r.reward
            is_prefix = r.is_prefix
            is_correctness = r.is_correctness
            traj = {'traj_vect': traj_vect, 'score': score, 'is_prefix':
                is_prefix, 'is_correctness': is_correctness}
            trajs.append(traj)

    # for traj in trajs:
    #     print("{}\n".format(traj))
    # print("length of trajs", len(trajs))
    # print(type(trajs))

    # check if the data satisfy that all is_correctness equals false
    correctness = False
    for traj in trajs:
        if traj.get('is_correctness'):
            correctness = True
    if correctness:
        print('Not all "is_correctness"are false')
        exit()

    score = np.array(list(traj['score'] for traj in trajs))

    behaviors = []
    for traj in trajs:
        inp = []
        for traj_vect in traj['traj_vect']:
            inp.append(traj_vect[0] + traj_vect[1])
        behaviors.append(inp)

    prefix = np.array(list(traj['is_prefix'] for traj in trajs))

    return score, behaviors, prefix


import pandas as pd
import re
import nltk
from nltk.stem.porter import PorterStemmer
import html
import inflect
from nltk.corpus import stopwords
import unicodedata
import contractions as c
from collections import Counter
import itertools
import operator
import numpy as np
import sys
import zipfile
import os
nltk.download("punkt")

def create_dataframe_imdb(file='../data/IMDB_Dataset.csv'):
    return pd.read_csv(file, names=['text', 'labels'], header=0)

def create_dataframe_bbc(zip_path='../data/bbc.zip'):
    datadir = './bbc'
    zip = zipfile.ZipFile(zip_path)
    zip.extractall()

    dirs = os.listdir(datadir)
    data = []
    for each in dirs:
        if each == 'README.TXT':
            continue
        sub_dir = os.listdir(datadir+'/'+each)
        for every in sub_dir:
            path = datadir+'/'+each+'/'+every
            with open(path, encoding='utf-8', errors='ignore') as f:
                text = f.read()
            data.append([text, each])
    return pd.DataFrame(data, columns=['text', 'label'])

def fixup(x):
    re1 = re.compile(r'  +')
    x = x.replace('#39;', "'"). \
        replace('amp;', '&'). \
        replace('#146;', "'"). \
        replace('nbsp;', ' '). \
        replace('#36;', '$'). \
        replace('\\n', "\n"). \
        replace('quot;', "'"). \
        replace('<br />', "\n"). \
        replace('\\"', '"'). \
        replace('<unk>', 'u_n'). \
        replace(' @.@ ', '.'). \
        replace(' @-@ ', '-'). \
        replace('\\', ' \\ '). \
        replace('>', ''). \
        replace('-', '')
    return re1.sub(' ', html.unescape(x))

def stem_tokens(tokens, stemmer):
    stemmed = []
    for item in tokens:
        stemmed.append(stemmer.stem(item))
    return stemmed

def tokenize(text):
    # tokenize
    tokens = nltk.word_tokenize(text)
    # Remove stop words
    tokens = [w for w in tokens if w not in stop_words]
    tokens = [w for w in tokens if len(w) >= 2]
    tokens = [fixup(x) for x in tokens]
    # stem
    # tokens = stem_tokens(tokens, stemmer)
    return tokens

def remove_non_ascii(words):
    """Remove non-ASCII characters from list of tokenized words"""
    new_words = []
    for word in words:
        new_word = unicodedata.normalize('NFKD', word). \
            encode('ascii', 'ignore'). \
            decode('utf-8', 'ignore')
        new_words.append(new_word)
    return new_words

def to_lowercase(words):
    """Convert all characters to lowercase from list of tokenized words"""
    new_words = []
    for word in words:
        new_word = word.lower()
        new_words.append(new_word)
    return new_words

def remove_punctuation(words):
    """Remove punctuation from list of tokenized words"""
    new_words = []
    for word in words:
        new_word = re.sub(r'[^\w\s]', '', word)
        if new_word != '':
            new_words.append(new_word)
    return new_words

def replace_numbers(words):
    """Replace all interger occurrences in list of
    tokenized words with textual representation"""
    p = inflect.engine()
    new_words = []
    for word in words:
        if word.isdigit():
            new_word = p.number_to_words(word)
            new_words.append(new_word)
        else:
            new_words.append(word)
    return new_words

def remove_stopwords(words):
    """Remove stop words from list of tokenized words"""
    new_words = []
    for word in words:
        if word not in stopwords.words('english'):
            new_words.append(word)
    return new_words

def contract(sent):
    ans = ''
    for each in sent.split():
        ans = ans + c.fix(each) + ' '
    return ans.strip()

def normalize(docs):
    docs = remove_non_ascii(to_lowercase([fixup(x) for x in docs]))
    docs = remove_stopwords(remove_punctuation([contract(x) for x in docs]))
    docs = [tokenize(x) for x in docs]
    print("Stage 1 complete.")
    sent = list(itertools.chain.from_iterable(docs))
    counter = Counter(sent)
    vocab = set(np.array(sorted(counter.items(),
                key=operator.itemgetter(1), reverse=True)[:10000])[:, 0])
    print("Vocab created.")
    docs = [[x for x in doc if x in vocab] for doc in docs]
    return docs

stemmer = PorterStemmer()
stop_words = set(stopwords.words('english'))

def clean_data(typ):
    if typ == 'imdb':
        dat = create_dataframe_imdb()
        dat.text = normalize(dat.text)
        dat.to_csv('../data/cleaned_documents_imdb.csv')

    else:
        dat = create_dataframe_bbc()
        dat.text = normalize(dat.text)
        dat.to_csv('../data/cleaned_documents.csv')

if __name__ == "__main__":
    if len(sys.argv) > 1:
        clean_data(sys.argv[1])
    else:
        clean_data("bbc")


import shutil
import os
import datetime

def backup_directory(source_dir, destination_dir):
    # Create a new directory with the current date and time
    current_time = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
    backup_dir = os.path.join(destination_dir, f"backup_{current_time}")

    # Copy the contents of the source directory to the backup directory
    try:
        shutil.copytree(source_dir, backup_dir)
        print(f"Backup created successfully at: {backup_dir}")
    except shutil.Error as e:
        print(f"Backup failed: {e}")

# Set the source and destination directories
source_directory = "/path/to/source/directory"
destination_directory = "/path/to/destination/directory"

# Call the backup_directory function
backup_directory(source_directory, destination_directory)


def unique(id,info_load):
    for id_array in info_load:
        if  id == id_array["id"]:
            return True
    return False

def judge(judge):
    while True:
        if judge=="yes":
            return True
        elif judge=="no":
            return False
        else:
            judge=input("yesかnoで答えてください")

def count(count,info_load,id):
    for id_array in info_load:
        if  id == id_array["id"]:
            return count
        count=count+1


def main():
    s = ""
    with open('input.txt', 'r') as f:
        s = [x.strip() for x in f.readlines()]

    my_tuple = (1, 2, 3)
    hm = {}
    squareCount = 0
    # print(s)

    for x in s:
        # print(x)
        [left, top] = x.split(",")
        [top, width] = top.split(": ")
        [width, height] = width.split("x")
        e = left.split("@ ")
        left = e[1]
        # print(left)
        # print(top)
        # print(width)
        # print(height)
        for y in range(int(width)):
            for z in range(int(height)):
                a = int(left) + y
                b = int(top) + z
                if (a,b) in hm:
                    hm[(a,b)] = hm[(a,b)] + 1
                else:
                    hm[(a,b)] = 1
        # x[1] = x.split(":")
        # x[2] = x.split("x")
        # print(x)
    count = 0
    for (k, v) in hm.items():
        if v > 1:
            count = count + 1
    print(count)

if __name__ == '__main__':
    main()


""" module: tasks

** Content **
A collection of thread-objects that perform certain hardware-bound tasks

** Details **
We have a Countdown, Buzzer, Key, Cursor aso Object here...

@author: (c) Thomas Lüth 2019 / info@tlc-it-consulting.com
@created: 2019-10-01
"""
from __future__ import absolute_import

from tlu_hardware import tlu_led,tlu_buttons,tlu_buzzer,tlu_matrix, tlu_cursor,\
    tlu_hardware_global, tlu_touch
import time
import logging
from tlu_services.tlu_queue import tlu_queueobject, tlu_queue
from threading import Thread
from tlu_game import tlu_globals
from datetime import datetime
from tlu_hardware.tlu_vibration import tlu_vibrate
from tlu_hardware.tlu_checkhardware import emulatekey
from queue import Empty
from tlu_hardware.tlu_hardwarebase import tlu_hardwarebase

logger = logging.getLogger(__name__)

class Countdown(Thread):
    """
    Provides a countdown display (tenths seconds) on 7-segment
    Once timer is run to zero a timeout message would be send to the provided queue
    """
    def __init__(self, queue, seconds):
        """
        Defines the queue where the timeout message would be send to and the amount of seconds the level should last
        :param queue: Message-Queue for Timeout message
        :param seconds: Seconds to wait until timeout, max 99 seconds allowed
        """
        Thread.__init__(self)
        self.is_aborted=False
        self.has_to_restart=False
        self.paused = seconds<0
        if seconds > 999:
            seconds = 999
        if seconds <= 0:
            seconds = 0.1
        self.tenths=int(seconds*10)
        self.queue=queue
        self.sevenseg=tlu_led.seven_segment()
        tlu_globals.init()
        tlu_hardware_global.init()
        glob=tlu_globals.globMgr.tlu_glob()
        glob.stopClock() #terminate Clock in case it is showing

    def run(self, *args, **kwargs):
        logger.info("Countdown started with tenths="+str(self.tenths)+" paused="+str(getattr(self, "paused", False)))
        t0=datetime.now()
        diff=0
        while diff < self.tenths:
            if getattr(self, "has_to_restart", False):
                t0=datetime.now()
                logger.info('Countdown restarted')
                diff=0
                self.has_to_restart=False
                self.paused=False
            if getattr(self, "paused", False):
                t0=datetime.now()
                diff=0
                self.sevenseg.clear()
                time.sleep(0.5)
                continue
            self.sevenseg.set4digits(int(self.tenths-diff), 0)
            time.sleep(0.1)
            t1=datetime.now()
            diff = int((t1-t0).total_seconds() * 10)
            if getattr(self, "is_aborted", False):
                self.sevenseg.clear()
                logger.info('Countdown aborted')
                return
        logger.info('Countdown ended with diff='+str(diff))
        queueobject=tlu_queueobject(tlu_queue.MSG_TIMEOUT)
        self.queue.send(queueobject)
        return Thread.run(self, *args, **kwargs)
    def terminate(self, *args, **kwargs):
        self.is_aborted=True
        logger.info('Countdown termination requested')
    def pause(self):
        self.paused=True
        logger.info("Countdown paused")
    def restart(self):
        self.has_to_restart=True
    def changeSecondsAndRestart(self, seconds):
        if seconds > 999:
            seconds = 999
        if seconds <= 0:
            seconds = 0.1
        self.tenths=int(seconds*10)
        self.has_to_restart=True

class CheckKey(Thread):
    """
    Check for any key pressed on the board. Once a key is pressed or released a message will be
    created and send to the provided queue
    """
    def __init__(self, queue, limit=None):
        """
        Initializes the Thread.
        :param queue: Queue to receive the key-messages or a tiemout , if set
        :param limit: Default none, else seconds to wait for a key before a timeout is sent to the queue
        """
        Thread.__init__(self)
        self.is_aborted=False
        self.buttons=tlu_buttons.tlu_buttons()
        if limit != None:
            self.count=int(limit*5) #to reach seconds
        else:
            self.count=None
        self.currentKey=0
        self.button_values=(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16)
        self.queue=queue

    def run(self, *args, **kwargs):
        logger.info("Checking for Key pressed in background")

        while True:
            if (self.count != None):
                self.count -= 1
                logger.debug("Requesting button with count = "+str(self.count))
                if self.count < 1:
                    queueobject=tlu_queueobject(tlu_queue.MSG_TIMEOUT)
                    self.queue.send(queueobject)
                    logger.info("Buttons no longer checked due to timeout")
                    return
            if getattr(self, "is_aborted", False):
                queueobject=tlu_queueobject(tlu_queue.MSG_STOP)
                self.queue.send(queueobject)
                logger.info("Buttons no longer checked due to abort")
                return
            time.sleep(0.1) #reduce load
            button=self.buttons.button_pressed()
            if button not in self.button_values:
                continue
            if button != self.currentKey:
                if (self.currentKey == 0) and (button > 0):
                    #pressed
                    self.currentKey=button
                    queueobject=tlu_queueobject(tlu_queue.MSG_KEYPRESSED,button)
                    self.queue.send(queueobject)
                    logger.info("Key pressed: "+str(button))
                elif (self.currentKey > 0) and (button == 0):
                    #released
                    queueobject=tlu_queueobject(tlu_queue.MSG_KEYRELEASED,self.currentKey)
                    self.queue.send(queueobject)
                    logger.info("Key released: "+str(self.currentKey))
                    self.currentKey=button #indicate release
        return Thread.run(self, *args, **kwargs)
    def terminate(self, *args, **kwargs):
        self.is_aborted=True
        logger.info('CheckKey termination requested')

class Buzzer(Thread):
    """
    Make the buzzer sound
    """
    def __init__(self,duration_seconds):
        """
        Initialization
        :param duration_seconds: Duration of sound in seconds
        """
        Thread.__init__(self)
        self.is_aborted=False
        self.tenth=10*duration_seconds
        self.buz=tlu_buzzer.tlu_buz()

    def run(self, *args, **kwargs):
        logger.info('Buzzer started with duration='+str(self.tenth))
        self.buz.sound(True)
        while (self.tenth > 0) and (not getattr(self, "is_aborted", False)):
            time.sleep(0.1)
            self.tenth -= 1
        self.buz.sound(False)
        return Thread.run(self, *args, **kwargs)
    def terminate(self, *args, **kwargs):
        self.is_aborted=True
        self.buz.sound(False)
        logger.info('Buzzer termination requested')

class AnimatedBuzzer(Thread):
    """
    Buzzer with symbols shown on the matrix.
    You see a note while the buzzer sounds, followed by a space once done
    """
    def __init__(self, duration_seconds):
        """
        Initialization
        :param duration_seconds: Duration of the sound in seconds
        """
        Thread.__init__(self)
        self.is_aborted=False
        logger.info('animated buzzer started with duration='+str(duration_seconds))
        self.matrix=tlu_matrix.tlu_matrix()
        self.matrix.brightness(2)
        self.matrix.show_symbol('sound')
        self.buz=tlu_buzzer.tlu_buz()
        self.tenth=duration_seconds*10

    def run(self):
        logger.info('animated buzzer started with duration='+str(self.tenth))
        self.buz.sound(True)
        while (self.tenth > 0):
            time.sleep(0.1)
            self.tenth -= 1
            if getattr(self, "is_aborted", False):
                logger.debug("Aborting AnimatedBuzzer")
                break
        self.buz.sound(False)
        if getattr(self, "is_aborted", False):
            self.matrix.show_symbol('space')
        else:
            self.matrix.show_symbol('root')
        logger.info('animated buzzer ended')
    def terminate(self, *args, **kwargs):
        self.is_aborted=True
        self.buz.sound(False)
        logger.info('animated buzzer termination requested')

class CheckCursor(Thread):
    """
    Checks for cursor-key input
    """
    def __init__(self, queue, limit=None):
        """
        Initializes the Thread.
        :param queue: Queue to receive the cursorkey-messages or a tiemout , if set
        :param limit: Default none, else seconds to wait for a cursorkey before a timeout is sent to the queue
        """
        Thread.__init__(self)
        self.is_aborted=False
        self.cursorkeys=tlu_cursor.tlu_cursor()
        if limit != None:
            self.count=int(limit*10)
        else:
            self.count=None
        self.currentKey=0
        self.cursor_values=(0,100,200,300,400)
        self.queue=queue
    def run(self, *args, **kwargs):
        logger.info("Checking for Cursor-Key pressed in background")
        while True:
            if (self.count != None):
                self.count -= 1
                if self.count < 1:
                    logger.info("exiting cursor-check due to retry-limit reached")
                    queueobject=tlu_queueobject(tlu_queue.MSG_TIMEOUT)
                    self.queue.send(queueobject)
                    return
            time.sleep(0.1)
            if getattr(self, "is_aborted", False):
                logger.info("Cursor-Keys no longer checked")
                queueobject=tlu_queueobject(tlu_queue.MSG_STOP)
                self.queue.send(queueobject)
                return
            if self.count != None:
                logger.debug("requesting cursor key  with count "+str(self.count))
            cursorkey=self.cursorkeys.cursor_pressed()
            if cursorkey not in self.cursor_values:
                continue
            if cursorkey != self.currentKey:
                if (self.currentKey >= 0) and (cursorkey == 0):
                    #released
                    queueobject=tlu_queueobject(tlu_queue.MSG_KEYRELEASED,self.currentKey)
                    self.queue.send(queueobject)
                    logging.info("Cursor released: "+str(self.currentKey))
                if (cursorkey > 0):
                    #pressed
                    queueobject=tlu_queueobject(tlu_queue.MSG_KEYPRESSED,cursorkey)
                    self.queue.send(queueobject)
                    logging.info("Cursor presssed: "+str(cursorkey))
                self.currentKey=cursorkey
        return Thread.run(self, *args, **kwargs)
    def terminate(self, *args, **kwargs):
        self.is_aborted=True
        logger.info('CheckCursor termination requested')

class Vibrate(Thread):
    """
    Make tvibration
    """
    def __init__(self,duration_seconds):
        """
        Initialization
        :param duration_seconds: Duration of sound in seconds
        """
        Thread.__init__(self)
        self.is_aborted=False
        self.tenth=10*duration_seconds
        self.vib=tlu_vibrate()

    def run(self, *args, **kwargs):
        logger.info('Vibration started with duration='+str(self.tenth))
        self.vib.vibrate(True)
        while (self.tenth > 0) and (not getattr(self, "is_aborted", False)):
            time.sleep(0.1)
            self.tenth -= 1
        self.vib.vibrate(False)
        return Thread.run(self, *args, **kwargs)
    def terminate(self, *args, **kwargs):
        self.is_aborted=True
        self.vib.vibrate(False)
        logger.info('vibration termination requested')

touch_queue=None
def do_touch_check(channel):
    global touch_queue
    queueobject=tlu_queueobject(tlu_queue.MSG_TOUCH_PRESSED)
    touch_queue.send(queueobject)
    logging.info("Touchpad pressed ")

class CheckTouch(Thread):
    """
    Checks for cursor-key input
    """
    def __init__(self, queue, limit=None):
        """
        Initializes the Thread.
        :param queue: Queue to receive the touchpad-messages or a tiemout , if set
        :param limit: Default none, else seconds to wait for a touchpad-event before a timeout is sent to the queue
        """
        global touch_queue
        Thread.__init__(self)
        self.is_aborted=False
        self.touch=tlu_touch.tlu_touch()
        if limit != None:
            self.count=int(limit*10)
        else:
            self.count=None
        self.queue=queue
        touch_queue=queue
    def run(self, *args, **kwargs):
        logger.info("Checking for Touchpad touched in background")
        self.touch.start(do_touch_check)
        while True:
            if (self.count != None):
                self.count -= 1
                if self.count < 1:
                    self.touch.stop()
                    logger.info("exiting touch-check due to retry-limit reached")
                    queueobject=tlu_queueobject(tlu_queue.MSG_TIMEOUT)
                    self.queue.send(queueobject)
                    return
            time.sleep(0.1)
            if getattr(self, "is_aborted", False):
                self.touch.stop()
                logger.info("Touchpad no longer checked")
                queueobject=tlu_queueobject(tlu_queue.MSG_STOP)
                self.queue.send(queueobject)
                return
            if self.count != None:
                logger.debug("requesting touchpad pressed with count "+str(self.count))
            if emulatekey:
                q=tlu_globals.tqueue
                try:
                    queueobject = q.get(block=False,timeout=1)
                except Empty:
                    logger.debug('touch-q is empty :(')
                    continue
                except Exception as e:
                    logger.warn('Exception while reading from key-q: '+str(e))
                    queueobject=tlu_queueobject(tlu_queue.MSG_STOP)
                    self.queue.send(queueobject)
                    return
                if queueobject.msg_num == tlu_queue.MSG_TOUCH_PRESSED:
                    do_touch_check(tlu_hardwarebase.touch_bcm)
                q.task_done() #release object from queue
        return Thread.run(self, *args, **kwargs)
    def terminate(self, *args, **kwargs):
        self.touch.stop()
        self.is_aborted=True
        logger.info('CheckCursor termination requested')


PRINT_TIM    =  0b00000001
HALT         =  0b10  # 2
PRINT_NUM    =  0b00000011  # opcode 3
SAVE         =  0b100
PRINT_REG    =  0b101    # opcode 5
ADD          =  0b110

# registers[2] = registers[2] + registers[3]

memory = [
    PRINT_TIM,
    PRINT_TIM,
    PRINT_NUM,
    42,
    SAVE,
    2,       # register to put it in
    99,      # number to save
    SAVE,
    3,      # register to save in
    1,      # number to save
    ADD,
    2,   # register to look at, and save stuff in
    3,   # register to look at
    PRINT_REG,
    2,       # register to look at
    HALT,
          ]

# write a program to pull each command out of memory and execute
# We can loop over it!
#
# register aka memory
registers = [0] * 8
# [0,0,99,0,0,0,0,0]
# R0-R7

pc = 0  # program counter
running = True
while running:
    command = memory[pc]

    if command == PRINT_TIM:
        print("Tim!")

    if command == HALT:
        running = False

    if command == PRINT_NUM:
        num_to_print = memory[pc + 1]
        print(num_to_print)
        pc += 1

    if command == SAVE:
        reg = memory[pc + 1]
        num_to_save = memory[pc + 2]
        registers[reg] = num_to_save

        pc += 2

    if command == PRINT_REG:
        reg_index = memory[pc + 1]
        print(registers[reg_index])
        pc += 1

    if command == ADD:
        first_reg = memory[pc + 1]
        sec_reg = memory[pc + 2]
        registers[first_reg] = registers[first_reg] + registers[sec_reg]
        pc += 2

    pc += 1


from io import BytesIO
from pathlib import Path

from lxml import etree

from .report_generator import ReportGenerator

XSL_DIR = Path(__file__).parent / "xsl"

class OldStyleHTMLReportGenerator(ReportGenerator):
    def __init__(self, parser):  # pylint: disable=W0231
        self.xml_report = parser.root
        self.xslt_doc = etree.parse(str(XSL_DIR / "xccdf-report.xsl"))
        self.xslt_transformer = etree.XSLT(self.xslt_doc)

    def generate_report(self, debug_setting):
        html_report = self.xslt_transformer(self.xml_report)
        result_html = etree.tostring(
            html_report,
            xml_declaration=True,
            doctype="<!DOCTYPE html>",
            encoding="utf-8",
            standalone=False,
            with_tail=False,
            method="html",
            pretty_print=debug_setting.no_minify)
        return BytesIO(result_html)


#!/usr/bin/env python3
""" Two threads cooking soup """

import threading
import time

class ChefDenem(threading.Thread):
    def __init__(self):
        super().__init__()

    def run(self):
        print('Denem is going to help with the sauce')
        time.sleep(2)
        print('Denem is done with the sauce')

class ChefOlivia(threading.Thread):
    def __init__(self):
        super().__init__()

    def run(self):
        print('Olivia started & waiting for sausage to thaw...')
        time.sleep(3)
        print('Olivia is done cutting sausage.')

# main thread a.k.a. Barron
if __name__ == '__main__':
    print("Barron started & requesting Olivia's help.")
    olivia = ChefOlivia()

    print('Barron is asking Denem to make the sauce')
    denem = ChefDenem()

    print('Barron asks Denem to prep the sauce')
    denem.start()
    denem.join()

    print('Barron tells Olivia to explicitly start.')
    olivia.start()

    print('Barron continues cooking soup.')
    time.sleep(0.5)

    print('Barron patiently waits for Olivia and Denem to finish and join...')
    olivia.join()

    print('Barron and Olivia and Denem are all done!')

Lesson 10 - Concurrent and Parallel Programming/Lesson set 1/CH02/02_09/begin/thread_lifecycle.py


from typing import List
class ListNode:
    def __init__(self,x):
        self.val = x
        self.next = None

class Solution:
    def mergeKLists(self, lists: List[ListNode]) -> ListNode:
        nodes = []
        head = point = ListNode(0)
        for l in lists:
            while l:
                nodes.append(l.val)
                l = l.next
        for x in sorted(nodes):
            point.next = ListNode(x)
            point = point.next
        return head.next

array1 = [1,4,5]
array2 = [1,3,4]
array3 = [2,6]
L1 = ListNode(None)
L2 = ListNode(None)
L3 = ListNode(None)
for i,val in enumerate(array1):
    if i == 0:
        L1 = ListNode(val)
        v1 = L1
    else:
        v1.next = ListNode(val)
        v1 = v1.next
for i,val in enumerate(array2):
    if i == 0:
        L2 = ListNode(val)
        v2 = L2
    else:
        v2.next = ListNode(val)
        v2 = v2.next
for i,val in enumerate(array3):
    if i == 0:
        L3 = ListNode(val)
        v3 = L3
    else:
        v3.next = ListNode(val)
        v3 = v3.next

L = [L1,L2,L3]
sol = Solution()
result = sol.mergeKLists(L)
a = 1

#L1 = [1->4->5,1->3->4,2->6]


from itertools import permutations

perm = permutations([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
for k in range(1_000_000):
    x = next(perm)
print(''.join([str(num) for num in x]))

# #### Using own functions #########
# def getPermutations(array):
#     output = []
#     getPermutationsHelper(array, [], output)
#     return output
#
#
# def getPermutationsHelper(array, perm, output):
#     if not len(array) and len(perm):
#         output.append(perm)
#     else:
#         for j in range(len(array)):
#             new_perm = perm + [array[j]]
#             new_array = array.copy()
#             new_array.pop(j)
#             getPermutationsHelper(new_array, new_perm, output)
#
#
# perm = getPermutations([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
# print(''.join([str(num) for num in perm[1_000_000]]))


from peewee import *
import pymysql
import datetime
from playhouse.shortcuts import RetryOperationalError

db = SqliteDatabase('musicbot.db')

class Song(Model):
    title = CharField(max_length=100, null=True)
    url = CharField(max_length=100, null=False)
    length = IntegerField(null=True)
    plays = IntegerField(default=0)

class Playlist(Model):
    name = CharField()

class Mapping(Model):
    song = ForeignKeyField(Song)
    playlist = ForeignKeyField(Playlist)

class Operator(Model):
    user_id = BigIntegerField(null=False)

class Server(Model):
    server_id = BigIntegerField(null=False)
    music_channel = BigIntegerField(null=True)
    chat_channel = BigIntegerField(null=True)
    autojoin = BooleanField(null=False, default=False)


import json
import re
import traceback
from pathlib import Path

from nltk.corpus import stopwords
from nltk.corpus import wordnet as wn
from sekg.constant.code import CodeEntityCategory, CodeEntityRelationCategory
from sekg.constant.constant import PropertyConstant, DomainConstant, OperationConstance
from sekg.graph.exporter.graph_data import GraphData
from sekg.term.fusion import Fusion
from sekg.text.extractor.domain_entity.entity_extractor import EntityExtractor
from sekg.text.extractor.domain_entity.identifier_util import IdentifierInfoExtractor
from sekg.text.extractor.domain_entity.relation_detection import RelationType, RelationDetector
from sekg.util.code import ConceptElementNameUtil

class DomainKGFusion:
    """
    build the skeleton KG from the JavaParser analysis result for the Project Source Code.
    It will include the package, class, interface, method.
    """

    STOPLIST = set(stopwords.words('english'))

    METHOD_LABELS = {
        CodeEntityCategory.to_str(CodeEntityCategory.CATEGORY_METHOD),
        CodeEntityCategory.to_str(CodeEntityCategory.CATEGORY_BASE_OVERRIDE_METHOD),
    }

    CLASS_LABELS = {
        CodeEntityCategory.to_str(CodeEntityCategory.CATEGORY_CLASS),
        CodeEntityCategory.to_str(CodeEntityCategory.CATEGORY_PACKAGE),
        CodeEntityCategory.to_str(CodeEntityCategory.CATEGORY_INTERFACE),
        CodeEntityCategory.to_str(CodeEntityCategory.CATEGORY_ENUM_CONSTANTS),
    }

    VARIABLE_LABELS = {
        CodeEntityCategory.to_str(CodeEntityCategory.CATEGORY_FIELD),
        CodeEntityCategory.to_str(CodeEntityCategory.CATEGORY_LOCAL_VARIABLE),
        CodeEntityCategory.to_str(CodeEntityCategory.CATEGORY_PARAMETER),
        CodeEntityCategory.to_str(CodeEntityCategory.CATEGORY_FIELD_OF_CLASS),
    }

    def __init__(self):
        self.graph_data = GraphData()
        self.text_extractor = EntityExtractor()
        self.detector = RelationDetector()
        self.identifier_info_extractor = IdentifierInfoExtractor()

    def init_graph_data(self, graph_data_path):
        self.graph_data = GraphData.load(graph_data_path)

    def add_code_relation(self, start_node_id, relation_name, code_element):
        name = code_element.split("<")[0].split(".")[-1].split(" ")[-1]
        if len(name) == 0:
            return
        node_json = self.graph_data.find_one_node_by_property(PropertyConstant.ALIAS, name)
        if node_json is None:
            return
        end_node_id = node_json[GraphData.DEFAULT_KEY_NODE_ID]
        self.graph_data.add_relation(startId=start_node_id,
                                     relationType=relation_name,
                                     endId=end_node_id)

    def handle_comment_in_class(self, node_id, node_properties):
        terms = set()
        linkages = set()

        comment = node_properties.get(PropertyConstant.COMMENT, "")
        domain_terms, code_elements = self.text_extractor.extract_from_comment(comment)
        for term in domain_terms:
            terms.add(term)
            linkages.add((node_id, RelationType.MENTION_IN_COMENT.value, term))

        return terms, linkages

    def handle_text_in_method(self, node_id, node_properties):
        terms = set()
        linkages = set()

        comment = node_properties.get(PropertyConstant.COMMENT, "")
        domain_terms, code_elements = self.text_extractor.extract_from_comment(comment)
        for term in domain_terms:
            terms.add(term)
            linkages.add((node_id, RelationType.MENTION_IN_COMENT.value, term))
        for element in code_elements:
            self.add_code_relation(node_id, RelationType.MENTION_IN_COMENT.value, element)

        for inside_comment in node_properties.get(PropertyConstant.INSIDE_COMMENT, []):
            domain_terms, code_elements = self.text_extractor.extract_from_sentence(inside_comment)
            for term in domain_terms:
                terms.add(term)
                linkages.add((node_id, RelationType.MENTION_IN_INSIDE_COMENT.value, term))
            for element in code_elements:
                self.add_code_relation(node_id, RelationType.MENTION_IN_INSIDE_COMENT.value, element)

        for literal_expr in node_properties.get(PropertyConstant.STRING_LITERAL_EXPR, []):

            domain_terms, code_elements = self.text_extractor.extract_from_comment(literal_expr)
            for term in domain_terms:
                terms.add(term)
                linkages.add((node_id, RelationType.MENTION_IN_STRING_LITERAL.value, term))
            for element in code_elements:
                self.add_code_relation(node_id, RelationType.MENTION_IN_STRING_LITERAL.value, element)

        return terms, linkages

    def handle_description(self, node_id, description):
        terms = set()
        linkages = set()

        domain_terms, code_elements = self.text_extractor.extract_from_sentence(description)
        for term in domain_terms:
            linkages.add((node_id, RelationType.MENTION_IN_SHORT_DESCRIPTION.value, term))
        for element in code_elements:
            self.add_code_relation(node_id, RelationType.MENTION_IN_SHORT_DESCRIPTION.value, element)
        return terms, linkages

    def handle_method_name(self, node_id, name):

        terms, operations, relations, linkages = self.identifier_info_extractor.extract_from_method_name(
            name, mark_for_identifier_in_relation=node_id)

        belong_to_relations = self.graph_data.get_relations(node_id, CodeEntityRelationCategory.to_str(
            CodeEntityRelationCategory.RELATION_CATEGORY_BELONG_TO))
        if len(belong_to_relations) > 0:
            class_id = belong_to_relations.pop()[2]
            for op in operations:
                linkages.add((class_id, RelationType.HAS_OPERATION.value, op))

        return terms, operations, relations, linkages

    def handle_class_name(self, node_id, name):
        terms, relations, linkages = self.identifier_info_extractor.extract_from_class_name(name,
                                                                                mark_for_identifier_in_relation=node_id)

        return terms, relations, linkages

    def handle_variable_name(self, node_id, name):
        terms, relations, linkages = self.identifier_info_extractor.extract_from_variable(name,
                                                                                mark_for_identifier_in_relation=node_id)

        return terms, relations, linkages

    def extract_term_and_relation(self, term_save_path=None, operation_save_path=None, term_relation_save_path=None,
                                  linkage_save_path=None, term_aliases_save_path=None, not_fused_term_save_path=None):
        print("start extract term and relation from graph data")
        self.graph_data.print_graph_info()

        # cache the map for adding relation
        not_fused_terms = set()
        operations = set()
        relations = set()
        linkages = set()
        i = 0
        for node_id in list(self.graph_data.get_node_ids()):
            try:
                i = i + 1
                if (i % 100) == 0:
                    print("已经执行了%d次节点检索" % i)
                # else:
                #     i = i + 1
                node_json = self.graph_data.get_node_info_dict(node_id=node_id)
                if not node_json:
                    continue

                node_properties = node_json[GraphData.DEFAULT_KEY_NODE_PROPERTIES]
                node_labels = node_json[GraphData.DEFAULT_KEY_NODE_LABELS]

                # print(len(node_labels & self.METHOD_LABELS))

                if 'sentence' in node_labels:
                    terms_, linkages_ = self.handle_description(node_id, node_properties["sentence_name"])
                    not_fused_terms.update(terms_)
                    linkages.update(linkages_)
                    continue

                if len(node_labels & self.METHOD_LABELS) > 0:
                    terms_, linkages_ = self.handle_text_in_method(node_id, node_properties)
                    not_fused_terms.update(terms_)
                    linkages.update(linkages_)

                    terms_, operations_, relations_, linkages_ = self.handle_method_name(node_id, node_properties[
                        GraphData.DEFAULT_KEY_PROPERTY_QUALIFIED_NAME])
                    not_fused_terms.update(terms_)
                    operations.update(operations_)
                    relations.update(relations_)
                    linkages.update(linkages_)

                description = node_properties.get(PropertyConstant.DESCRIPTION, "")
                if description is not None and len(description) > 0:
                    terms_, linkages_ = self.handle_description(node_id, node_properties[PropertyConstant.DESCRIPTION])
                    not_fused_terms.update(terms_)
                    linkages.update(linkages_)

                if len(node_labels & self.CLASS_LABELS) > 0:
                    terms_, linkages_ = self.handle_comment_in_class(node_id, node_properties)
                    not_fused_terms.update(terms_)
                    linkages.update(linkages_)

                    terms_, relations_, linkages_ = self.handle_class_name(node_id, node_properties[
                        GraphData.DEFAULT_KEY_PROPERTY_QUALIFIED_NAME])
                    not_fused_terms.update(terms_)
                    linkages.update(linkages_)
                    relations.update(relations_)

                if len(node_labels & self.VARIABLE_LABELS) > 0:
                    terms_, linkages_ = self.handle_comment_in_class(node_id, node_properties)
                    not_fused_terms.update(terms_)
                    linkages.update(linkages_)

                    terms_, relations_, linkages_ = self.handle_variable_name(node_id, node_properties[
                        GraphData.DEFAULT_KEY_PROPERTY_QUALIFIED_NAME])
                    not_fused_terms.update(terms_)
                    linkages.update(linkages_)
                    relations.update(relations_)

            except:
                traceback.print_exc()
        new_terms = []
        for term in not_fused_terms:
            if self.valid_term(term):
                new_terms.append(term)
        not_fused_terms = set(new_terms)

        relations_ = self.detector.detect_relation_by_starfix(not_fused_terms)
        relations.update(relations_)

        print("complete domain extraction")
        term_fusion = Fusion()
        synsets = term_fusion.fuse_by_synonym(not_fused_terms)
        print("complete synonym fusion")

        fused_term_to_aliases_map = {}
        for synset in synsets:
            fused_term_to_aliases_map[synset.key] = list(synset.terms)

        fused_terms = fused_term_to_aliases_map.keys()

        new_relations = set()
        new_linkages = set()

        for start_e, relation_name, end_e in relations:
            if relation_name == "has operation":
                continue
            if relation_name == "can be operated":
                continue
            new_start_e_list = set()
            new_end_e_list = set()

            for fused_term, aliases in fused_term_to_aliases_map.items():
                if end_e in aliases:
                    new_end_e_list.add(fused_term)
                if start_e in aliases:
                    new_start_e_list.add(fused_term)

            for new_start_e in new_start_e_list:
                for new_end_e in new_end_e_list:
                    new_relations.add((new_start_e, relation_name, new_end_e))

            if len(new_start_e_list) == 0:
                new_start_e_list.add(start_e)
            if len(new_end_e_list) == 0:
                new_end_e_list.add(end_e)
        relations = new_relations

        for start_e, relation_name, end_e in linkages:
            if relation_name == "has operation":
                continue
            if relation_name == "can be operated":
                continue

            new_start_e_list = set()
            new_end_e_list = set()

            for fused_term, aliases in fused_term_to_aliases_map.items():
                if end_e in aliases:
                    new_end_e_list.add(fused_term)
                if start_e in aliases:
                    new_start_e_list.add(fused_term)
            if len(new_start_e_list) == 0:
                new_start_e_list.add(start_e)
            if len(new_end_e_list) == 0:
                new_end_e_list.add(end_e)

            for new_start_e in new_start_e_list:
                for new_end_e in new_end_e_list:
                    new_linkages.add((new_start_e, relation_name, new_end_e))

        linkages = new_linkages

        print("length of new_linkages %d" % (len(linkages)))

        # term_orgin = {k: list(v) for k, v in term_orgin.items()}

        import json

        if term_save_path is not None:
            with Path(term_save_path).open("w") as f:
                f.write("\n".join(sorted(fused_terms, key=lambda x: x)))

        if not_fused_term_save_path is not None:
            with Path(not_fused_term_save_path).open("w") as f:
                f.write("\n".join(sorted(not_fused_terms, key=lambda x: x)))

        if operation_save_path is not None:
            with Path(operation_save_path).open("w") as f:
                f.write("\n".join(sorted(operations, key=lambda x: x)))

        if term_relation_save_path is not None:
            with Path(term_relation_save_path).open("w") as f:
                json.dump(
                    [(r[0], str(r[1]), r[2]) for r in relations if self.valid_term(r[0]) and self.valid_term(r[2])], f,
                    indent=4)

        if linkage_save_path is not None:
            with Path(linkage_save_path).open("w") as f:
                json.dump([(r[0], str(r[1]), r[2]) for r in linkages if self.valid_term(r[0]) or self.valid_term(r[2])],
                          f, indent=4)

        if term_aliases_save_path is not None:
            with Path(term_aliases_save_path).open("w") as f:
                json.dump(fused_term_to_aliases_map,
                          f, indent=4)

        return fused_terms, operations, relations, linkages, fused_term_to_aliases_map

    def select_name(self, terms):
        return min(terms, key=lambda x: len(x))

    def valid_term(self, term):
        term = str(term)
        if len(term) <= 2 or term.isdigit() or (len(term) > 30 and len(term.split()) > 4):
            return False
        prefix, *rest = term.split()
        if prefix in self.STOPLIST:
            return False
        return True

    def add_domain_term(self, term, lemma, aliases):
        """
        add a new term to graph data
        :param term: the term added to GraphData
        :return: the node_id fo the added term node
        """
        if aliases == None:
            aliases = set([])
        else:
            aliases = set(list(aliases))
        aliases.add(lemma)
        aliases.add(term)

        node_labels = [DomainConstant.LABEL_DOMAIN_TERM]
        node_properties = {
            DomainConstant.PRIMARY_PROPERTY_NAME: term,
            PropertyConstant.ALIAS: aliases,
            PropertyConstant.LEMMA: lemma
        }
        domain_term_node_id = self.graph_data.add_node(node_labels=node_labels,
                                                       node_properties=node_properties,
                                                       primary_property_name=DomainConstant.PRIMARY_PROPERTY_NAME)
        return domain_term_node_id

    def add_operation(self, op, lemma):
        node_labels = [OperationConstance.LABEL_OPERATION]
        node_properties = {
            OperationConstance.PRIMARY_PROPERTY_NAME: op,
            PropertyConstant.ALIAS: {op},
            PropertyConstant.LEMMA: lemma
        }
        operation_node_id = self.graph_data.add_node(node_labels=node_labels,
                                                     node_properties=node_properties,
                                                     primary_property_name=OperationConstance.PRIMARY_PROPERTY_NAME)
        return operation_node_id

    def update_domain_node_alias(self, node_id, term):
        node_json = self.graph_data.get_node_info_dict(node_id=node_id)
        if not node_json:
            return
        node_properties = node_json[GraphData.DEFAULT_KEY_NODE_PROPERTIES]
        alias = node_properties[PropertyConstant.ALIAS]
        alias.add(term)
        name = self.select_name(alias)
        node_json[DomainConstant.PRIMARY_PROPERTY_NAME] = name
        self.graph_data.update_node_index(node_id=node_id)

    def update_operation_node_alias(self, node_id, term):
        node_json = self.graph_data.get_node_info_dict(node_id=node_id)
        if not node_json:
            return
        node_properties = node_json[GraphData.DEFAULT_KEY_NODE_PROPERTIES]
        alias = node_properties[PropertyConstant.ALIAS]
        alias.add(term)
        name = self.select_name(alias)
        node_json[OperationConstance.PRIMARY_PROPERTY_NAME] = name
        self.graph_data.update_node_index(node_id=node_id)

    def add_relation_for_same_name_operation_and_domain_term(self):
        operation_node_ids = self.graph_data.get_node_ids_by_label(OperationConstance.LABEL_OPERATION)

        for operation_id in operation_node_ids:
            operation_node = self.graph_data.get_node_info_dict(operation_id)

            operation_name = operation_node[GraphData.DEFAULT_KEY_NODE_PROPERTIES][
                OperationConstance.PRIMARY_PROPERTY_NAME]

            domain_term_node = self.graph_data.find_one_node_by_property(
                property_name=DomainConstant.PRIMARY_PROPERTY_NAME, property_value=operation_name)

            if domain_term_node == None:
                continue
            domain_term_node_id = domain_term_node[GraphData.DEFAULT_KEY_NODE_ID]

            self.graph_data.add_relation(operation_id, "corresponding concept", domain_term_node_id)
            self.graph_data.add_relation(domain_term_node_id, "corresponding operation", operation_id)

    def save(self, graph_data_path):
        self.graph_data.save(graph_data_path)

    def fuse(self, terms, operations, relations, linkages, aliases_map):
        """
        start import the term and their relation to graph
        :param term_origins:
        :param term_relations:
        :return:
        """
        # domain_graph_data = GraphData()

        self.graph_data.create_index_on_property(DomainConstant.PRIMARY_PROPERTY_NAME)
        self.graph_data.create_index_on_property(OperationConstance.PRIMARY_PROPERTY_NAME)
        self.graph_data.create_index_on_property(PropertyConstant.ALIAS)

        # todo:update the index when add
        print("start fuse with domain knowledge")
        self.graph_data.print_graph_info()

        term_lemma2id = {}
        term_name2id = {}
        op_lemma2id = {}
        op_name2id = {}

        def __add_or_update(name, is_op=False):
            if is_op:
                lemma = name.lower()
                if lemma in op_lemma2id:
                    node_id = op_lemma2id[lemma]
                    self.update_operation_node_alias(node_id, name)
                else:
                    node_id = self.add_operation(name, lemma)
                    if node_id == GraphData.UNASSIGNED_NODE_ID:
                        print("adding operation %r fail" % name)
                        return node_id
                    op_lemma2id[lemma] = node_id
                op_name2id[name] = node_id
            else:
                lemma = name.replace("-", " ").replace("  ", " ").lower()
                lemma = re.sub('([^v])([0-9]+)', r'\1 \2', lemma)
                node_id = self.add_domain_term(name, lemma, aliases=aliases_map.get(name, None))

                if node_id == GraphData.UNASSIGNED_NODE_ID:
                    print("adding domain term %r fail" % name)
                    return node_id
                term_lemma2id[lemma] = node_id
                term_name2id[name] = node_id
            return node_id

        for term in sorted(terms, key=lambda x: len(x.split())):
            __add_or_update(term)

        for op in operations:
            __add_or_update(op, is_op=True)

        def __add_relation(start_term, relation_name, end_term):
            start_name2id = term_name2id
            end_name2id = term_name2id
            start_term_is_op = False
            end_term_is_op = False
            # if relation_name.startswith("operation_"):
            #     start_name2id = op_name2id
            #     start_term_is_op = True
            if relation_name == "has operation":
                end_name2id = op_name2id
                end_term_is_op = True
            if relation_name == "instance of":
                end_name2id = op_name2id
                end_term_is_op = True
            if relation_name == "can be operated":
                end_name2id = op_name2id
                end_term_is_op = True

            if type(start_term) == int:
                start_node_id = start_term
            else:
                if start_term in start_name2id:
                    start_node_id = start_name2id[start_term]
                else:
                    start_node_id = __add_or_update(start_term, is_op=start_term_is_op)
                if start_node_id == GraphData.UNASSIGNED_NODE_ID:
                    print("adding start_domain term %r fail for relation %r" % (
                        start_term, (start_term, relation_name, end_term)))
                    return
            if type(end_term) == int:
                end_node_id = end_term
            else:
                if end_term in end_name2id:
                    end_node_id = end_name2id[end_term]
                else:
                    end_node_id = __add_or_update(end_term, is_op=end_term_is_op)
                if end_node_id == GraphData.UNASSIGNED_NODE_ID:
                    print("adding start_domain term %r fail for relation %r" % (
                        start_term, (start_term, relation_name, end_term)))
                    return

            self.graph_data.add_relation(startId=start_node_id,
                                         relationType=relation_name,
                                         endId=end_node_id)

        for (start_term, relation_name, end_term) in relations:
            __add_relation(start_term, relation_name, end_term)

        for (start_term, relation_name, end_term) in linkages:
            __add_relation(start_term, relation_name, end_term)

        isA_relations = set()
        for (start_id, _, end_id) in self.graph_data.get_relations(
                relation_type=CodeEntityRelationCategory.to_str(CodeEntityRelationCategory.RELATION_CATEGORY_EXTENDS)):
            start_domain_ids = {e for _, _, e in self.graph_data.get_relations(start_id=start_id,
                                                                               relation_type=RelationType.REPRESENT.value)}
            end_domain_ids = {e for _, _, e in self.graph_data.get_relations(start_id=start_id,
                                                                             relation_type=RelationType.REPRESENT.value)}
            for s in start_domain_ids:
                for e in end_domain_ids:
                    isA_relations.add((s, RelationType.IS_A.value, e))
        for r in isA_relations:
            __add_relation(*r)

        self.add_relation_for_same_name_operation_and_domain_term()
        print("end fuse with domain knowledge")
        self.graph_data.refresh_indexer()
        self.graph_data.print_graph_info()

    def build_aliases_for_domain_term_and_operations(self, new_all_aliases_save_path=None):
        name_util = ConceptElementNameUtil()
        domain_term_ids = self.graph_data.get_node_ids_by_label(DomainConstant.LABEL_DOMAIN_TERM)
        term_name_list = []
        fused_term_to_aliases_map = {}

        for domain_term_id in domain_term_ids:
            node_json = self.graph_data.get_node_info_dict(domain_term_id)
            term_name = node_json[GraphData.DEFAULT_KEY_NODE_PROPERTIES][DomainConstant.PRIMARY_PROPERTY_NAME]
            term_name_list.append(term_name)

        for domain_term_id in domain_term_ids:
            node_json = self.graph_data.get_node_info_dict(domain_term_id)
            term_name = node_json[GraphData.DEFAULT_KEY_NODE_PROPERTIES][DomainConstant.PRIMARY_PROPERTY_NAME]

            all_aliases_list = set([])

            generated_aliases = name_util.generate_aliases(term_name, vocabulary=term_name_list)
            all_aliases_list = all_aliases_list | set(generated_aliases)

            exist_aliases = node_json[GraphData.DEFAULT_KEY_NODE_PROPERTIES].get(PropertyConstant.ALIAS, set([]))

            for alias in exist_aliases:
                all_aliases_list.add(alias)
                generated_aliases = name_util.generate_aliases(term_name, vocabulary=term_name_list)
                all_aliases_list = all_aliases_list | set(generated_aliases)

            node_json[GraphData.DEFAULT_KEY_NODE_PROPERTIES][PropertyConstant.ALIAS] = all_aliases_list
            fused_term_to_aliases_map[term_name] = list(all_aliases_list)

        operation_ids = self.graph_data.get_node_ids_by_label(OperationConstance.LABEL_OPERATION)
        # todo: build the relation between operation and domain term
        for operation_id in operation_ids:
            node_json = self.graph_data.get_node_info_dict(operation_id)
            term_name = node_json[GraphData.DEFAULT_KEY_NODE_PROPERTIES][OperationConstance.PRIMARY_PROPERTY_NAME]

            synsets = wn.synsets(term_name, pos="v")
            generated_aliases = [synset.name().split(".")[0] for synset in synsets]

            exist_aliases = node_json[GraphData.DEFAULT_KEY_NODE_PROPERTIES].get(PropertyConstant.ALIAS, set([]))
            for alias in generated_aliases:
                exist_aliases.add(alias)

            node_json[GraphData.DEFAULT_KEY_NODE_PROPERTIES][PropertyConstant.ALIAS] = exist_aliases

        if new_all_aliases_save_path != None:
            with Path(new_all_aliases_save_path).open("w") as f:
                json.dump(fused_term_to_aliases_map,
                          f, indent=4)

        # todo: build the aliases for operation
        self.graph_data.refresh_indexer()

    def delete_islocated_nodes_by_label(self, label):
        domain_node_ids = self.graph_data.get_node_ids_by_label(label)
        remove_ids = []
        for domain_id in domain_node_ids:
            out_ids = self.graph_data.get_all_out_relations(domain_id)
            in_ids = self.graph_data.get_all_in_relations(domain_id)
            if not out_ids and not in_ids:
                remove_ids.append(domain_id)
        print("delete %d islocated domain term" % (len(remove_ids)))
        for id in remove_ids:
            # print("remove islocated node:", self.graph_data.get_node_info_dict(id))
            self.graph_data.remove_node(id)
        return self.graph_data

    def delete_nodes_and_relations(self, name_list):
        for name in name_list:
            node_info = self.graph_data.find_one_node_by_property(DomainConstant.PRIMARY_PROPERTY_NAME, name)
            if node_info:
                node_id = node_info["id"]
                out_relations = self.graph_data.get_all_out_relations(node_id)
                in_relations = self.graph_data.get_all_in_relations(node_id)
                for s, r, e in out_relations.union(in_relations):
                    # print('delete relation %d, %s, %d' % (s, r, e))
                    self.graph_data.remove_relation(s, r, e)
                self.graph_data.remove_node(node_id)
                # print("delete node %d" % (node_id))
            else:
                print("can't find node for %s" % (name))
        return self.graph_data


#!/usr/bin/env python
"""integrate.py: Demonstrate some SciPy integration functions.
"""
from scipy.integrate import trapz, simps, quad
import numpy as np

# pylint: disable=invalid-name

def f(t):
    """Integrand function."""

    return np.sin(t)

# Define sample points.
x = np.linspace(0, np.pi, num=100, endpoint=True)
y = f(x)

# Compute the integral by using trapezoid rule.
print("Trapezoid method: ", trapz(y, x))

# Compute the integral by using Simpsons's rule.
print("Simpson's method: ", simps(y, x))

# Compute the integral by using QUADPACK (Fortran).
# interval of integration; 0 to pi.
I3 = quad(f, x[0], x[-1])
print("QUADPACK: ", I3)


# -*- coding: utf-8 -*-
"""
Created on Sat Apr 11 12:19:08 2020

@author: vetur

from random import randint
"""

import matplotlib.pyplot as plt
from tkinter import *
from random import randint

root=Tk()
root.title("KBC")
label=Label(root,text="Lifelines").place(x=0,y=0)

def poll():

    ops=['A','B','C','D']
    perc=[20,0,0,0]
    for i in range(80):
        perc[randint(0,3)] += 1
    plt.bar(ops,perc,color='g')
    plt.legend()
    plt.xlabel("Options")
    plt.ylabel("Percentage")
    plt.title("Audience Poll")
    plt.show()

bt1=Button(root,text="Audience poll",command=poll)
bt1.pack()
bt1.place(x=0,y=25)

root.mainloop()


import unittest
from io import BytesIO
import os
import pandas as pd
import base64
from unittest.mock import MagicMock, patch
from typing import List

class TestExcelDownload(unittest.TestCase):
    def setUp(self):
        self.df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
        self.excel_data = to_excel(self.df)
        self.filename = 'data.xlsx'

    def test_get_table_download_link(self):
        expected_output = f'Download Excel File'
        self.assertEqual(get_table_download_link(self.excel_data, self.filename), expected_output)

    def test_convert_and_save_as_csv(self):
        # Test with a valid file
        uploaded_file = MagicMock(name='uploaded_file')
        uploaded_file.name = 'test.xlsx'
        uploaded_file.read.return_value = self.excel_data
        csv_file_path = convert_and_save_as_csv(uploaded_file)
        self.assertIsNotNone(csv_file_path)
        self.assertTrue(os.path.exists(csv_file_path))
        os.remove(csv_file_path)

        # Test with a None file
        self.assertIsNone(convert_and_save_as_csv(None))

        # Test with an invalid file
        uploaded_file = MagicMock(name='uploaded_file')
        uploaded_file.name = 'test.txt'
        self.assertIsNone(convert_and_save_as_csv(uploaded_file))

    def test_to_excel(self):
        expected_output = pd.read_excel(BytesIO(self.excel_data))
        self.assertTrue(self.df.equals(pd.read_excel(BytesIO(to_excel(self.df)))))

    def test_load_data(self):
        # Test with a valid file
        csv_file_path = 'test.csv'
        self.df.to_csv(csv_file_path, index=False)
        expected_output = pd.read_csv(csv_file_path)
        self.assertTrue(expected_output.equals(load_data(csv_file_path)))
        os.remove(csv_file_path)

        # Test with a None file
        self.assertIsNone(load_data(None))

        # Test with an invalid file
        self.assertIsNone(load_data('invalid_file.csv'))

if __name__ == '__main__':
    unittest.main()


from gocept.logging.testing import LogMessage
import gocept.logging
import logging
import unittest

class TestingHandlerTest(unittest.TestCase):

    def setUp(self):
        self.log = logging.getLogger('test')

    def tearDown(self):
        self.log.handlers[:] = []

    def test_inspect_extra_values(self):
        handler = gocept.logging.TestingHandler()
        self.log.setLevel(logging.DEBUG)
        self.log.addHandler(handler)

        self.log.warning('Hello, world!', extra={'foo': 'bar'})
        self.assertEqual(
            [LogMessage('test', 'WARNING', 'Hello, world!', {'foo': 'bar'})],
            handler.messages)


"""
Data and tree for the MG94 gene conversion example.

"""
from __future__ import division, print_function, absolute_import

import networkx as nx
from numpy.testing import assert_equal

__all__ = ['ad_hoc_fasta_reader', 'get_tree_info_with_outgroup']

def ad_hoc_fasta_reader(fin):
    name_seq_pairs = []
    while True:

        # read the name
        line = fin.readline().strip()
        if not line:
            return name_seq_pairs
        assert_equal(line[0], '>')
        name = line[1:].strip()

        # read the single line sequence
        line = fin.readline().strip()
        seq = line
        unrecognized = set(line) - set('ACGT')
        if unrecognized:
            raise Exception('unrecognized nucleotides: ' + str(unrecognized))

        name_seq_pairs.append((name, seq))

def get_tree_info_with_outgroup():
    T = nx.DiGraph()
    root = 'N0'
    T.add_edges_from([
            ('N0', 'kluyveri'),
            ('N0', 'N1'),
            ('N1', 'castellii'),
            ('N1', 'N2'),
            ('N2', 'bayanus'),
            ('N2', 'N3'),
            ('N3', 'kudriavzevii'),
            ('N3', 'N4'),
            ('N4', 'mikatae'),
            ('N4', 'N5'),
            ('N5', 'paradoxus'),
            ('N5', 'cerevisiae'),
            ])
    return T, root


# -*- coding: utf-8 -*-
"""
Created on Mon Aug 19 19:30:01 2019

@author: jacqueline.cortez

Capítulo 2. The Bias-Variance Tradeoff
Introduction:
    The bias-variance tradeoff is one of the fundamental concepts in supervised machine learning.
    In this chapter, you'll understand how to diagnose the problems of overfitting and underfitting.
    You'll also be introduced to the concept of ensembling where the predictions of several models are
    aggregated to produce predictions that are more robust.
"""

# Import packages
import pandas as pd                                                                 #For loading tabular data
#import numpy as np                                                                  #For making operations in lists
#import matplotlib as mpl                                                            #To format numbers with ax.yaxis.set_major_formatter(mpl.ticker.StrMethodFormatter('{x:,.0f}')) or ax.get_xaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
#import matplotlib.pyplot as plt                                                     #For creating charts
#import seaborn as sns                                                               #For visualizing data
#import scipy.stats as stats                                                         #For accesign to a vary of statistics functiosn
#import statsmodels as sm                                                            #For stimations in differents statistical models
#import scykit-learn                                                                 #For performing machine learning
#import tabula                                                                       #For extracting tables from pdf
#import nltk                                                                         #For working with text data
#import math                                                                         #For accesing to a complex math operations
#import random                                                                       #For generating random numbers
#import calendar                                                                     #For accesing to a vary of calendar operations
#import re                                                                           #For regular expressions
#import timeit                                                                       #For Measure execution time of small code snippets
#import time                                                                         #To measure the elapsed wall-clock time between two points
#import warnings
#import wikipedia

#from pandas.plotting import register_matplotlib_converters                          #For conversion as datetime index in x-axis
#from math import radian                                                             #For accessing a specific math operations
#from functools import reduce                                                        #For accessing to a high order functions (functions or operators that return functions)
#from pandas.api.types import CategoricalDtype                                       #For categorical data
#from glob import glob                                                               #For using with pathnames matching
#from datetime import datetime                                                       #For obteining today function
#from string import Template                                                         #For working with string, regular expressions
#from itertools import cycle                                                         #Used in the function plot_labeled_decision_regions()
#from math import floor                                                              #Used in the function plot_labeled_decision_regions()
#from math import ceil                                                               #Used in the function plot_labeled_decision_regions()

#from scipy.cluster.hierarchy import fcluster                                        #For learning machine - unsurpervised
#from scipy.cluster.hierarchy import dendrogram                                      #For learning machine - unsurpervised
#from scipy.cluster.hierarchy import linkage                                         #For learning machine - unsurpervised
#from scipy.sparse import csr_matrix                                                 #For learning machine
#from scipy.stats import pearsonr                                                    #For learning machine
#from scipy.stats import randint                                                     #For learning machine

#from sklearn.cluster import KMeans                                                  #For learning machine - unsurpervised
#from sklearn.decomposition import NMF                                               #For learning machine - unsurpervised
#from sklearn.decomposition import PCA                                               #For learning machine - unsurpervised
#from sklearn.decomposition import TruncatedSVD                                      #For learning machine - unsurpervised
#from sklearn.ensemble import RandomForestClassifier                                 #For learning machine
from sklearn.ensemble import VotingClassifier                                       #For learning machine - unsurpervised
#from sklearn.feature_extraction.text import TfidfVectorizer                         #For learning machine - unsurpervised
#from sklearn.feature_selection import chi2                                          #For learning machine
#from sklearn.feature_selection import SelectKBest                                   #For learning machine
#from sklearn.feature_extraction.text import CountVectorizer                         #For learning machine
#from sklearn.feature_extraction.text import HashingVectorizer                       #For learning machine
#from sklearn import datasets                                                        #For learning machine
#from sklearn.impute import SimpleImputer                                            #For learning machine
#from sklearn.linear_model import ElasticNet                                         #For learning machine
#from sklearn.linear_model import Lasso                                              #For learning machine
from sklearn.linear_model import LinearRegression                                   #For learning machine
from sklearn.linear_model import LogisticRegression                                 #For learning machine
#from sklearn.linear_model import Ridge                                              #For learning machine
#from sklearn.manifold import TSNE                                                   #For learning machine - unsurpervised
from sklearn.metrics import accuracy_score                                          #For learning machine
#from sklearn.metrics import classification_report                                   #For learning machine
#from sklearn.metrics import confusion_matrix                                        #For learning machine
from sklearn.metrics import mean_squared_error as MSE                               #For learning machine
#from sklearn.metrics import roc_auc_score                                           #For learning machine
#from sklearn.metrics import roc_curve                                               #For learning machine
from sklearn.model_selection import cross_val_score                                 #For learning machine
#from sklearn.model_selection import GridSearchCV                                    #For learning machine
#from sklearn.model_selection import RandomizedSearchCV                              #For learning machine
from sklearn.model_selection import train_test_split                                #For learning machine
#from sklearn.multiclass import OneVsRestClassifier                                   #For learning machine
from sklearn.neighbors import KNeighborsClassifier as KNN                            #For learning machine
#from sklearn.pipeline import FeatureUnion                                           #For learning machine
from sklearn.pipeline import make_pipeline                                          #For learning machine - unsurpervised
#from sklearn.pipeline import Pipeline                                               #For learning machine
#from sklearn.preprocessing import FunctionTransformer                               #For learning machine
#from sklearn.preprocessing import Imputer                                           #For learning machine
#from sklearn.preprocessing import MaxAbsScaler                                      #For learning machine (transforms the data so that all users have the same influence on the model)
#from sklearn.preprocessing import Normalizer                                        #For learning machine - unsurpervised (for pipeline)
#from sklearn.preprocessing import normalize                                         #For learning machine - unsurpervised
#from sklearn.preprocessing import scale                                             #For learning machine
from sklearn.preprocessing import StandardScaler                                    #For learning machine
#from sklearn.svm import SVC                                                         #For learning machine
from sklearn.tree import DecisionTreeClassifier                                     #For learning machine - supervised
from sklearn.tree import DecisionTreeRegressor                                      #For learning machine - supervised

#from bokeh.io import curdoc, output_file, show                                      #For interacting visualizations
#from bokeh.plotting import figure, ColumnDataSource                                 #For interacting visualizations
#from bokeh.layouts import row, widgetbox, column, gridplot                          #For interacting visualizations
#from bokeh.models import HoverTool, ColumnDataSource, CategoricalColorMapper        #For interacting visualizations
#from bokeh.models import Slider, Select, Button, CheckboxGroup, RadioGroup, Toggle  #For interacting visualizations
#from bokeh.models.widgets import Tabs, Panel                                        #For interacting visualizations
#from bokeh.palettes import Spectral6                                                #For interacting visualizations

# Setting the pandas options
#pd.set_option("display.max_columns",20)
#pd.options.display.float_format = '{:,.4f}'.format
#pd.reset_option("all")
#register_matplotlib_converters() #Require to explicitly register matplotlib converters.

#plt.rcParams = plt.rcParamsDefault
#plt.rcParams['figure.constrained_layout.use'] = True
#plt.rcParams['figure.constrained_layout.h_pad'] = 0.09

#Setting the numpy options
#np.set_printoptions(precision=3) #precision set the precision of the output:
#np.set_printoptions(suppress=True) #suppress suppresses the use of scientific notation for small numbers
#np.set_printoptions(threshold=np.inf) #Show all the columns and rows from an array.
#np.set_printoptions(threshold=8) #Return to default value.

#Setting images params
#plt.rcParams.update({'figure.max_open_warning': 0}) #To solve the max images open

print("****************************************************")
print("** BEGIN                                          **")
print("****************************************************")
print("** User defined functions \n")

SEED = 1

print("****************************************************")
print("** Getting the data for this program\n")

file = 'auto.csv'
auto_df = pd.read_csv(file)
auto_X = auto_df.drop(['mpg'], axis=1)
auto_X = pd.get_dummies(auto_X, prefix_sep='_')#, drop_first=True)
autoX_displ = auto_df.displ
auto_y = auto_df.mpg

file = 'indian_liver_patient.csv'
liver_df = pd.read_csv(file)
liver_df.dropna(inplace=True)
#liver_df['Gender'] = liver_df.Gender.map({'Female':0,'Male':1})
liver_df['Dataset'] = liver_df.Dataset.map({1: 1, 2: 0})
scaler = StandardScaler()
liver_df_standarized = liver_df.drop(['Gender','Dataset'], axis=1)
liver_df_standarized = pd.DataFrame(data=scaler.fit_transform(liver_df_standarized), columns=liver_df_standarized.columns)
liver_df_standarized['Is_Male'] = liver_df.Gender.map({'Female':0,'Male':1}).values
liver_df_standarized['Liver_Disease'] = liver_df['Dataset'].values
liver_X = liver_df_standarized.drop(['Liver_Disease'], axis=1)
liver_y = liver_df_standarized.Liver_Disease

print("****************************************************")
tema = "5. Instantiate the model"; print("** %s\n" % tema)

auto_X_train, auto_X_test, auto_y_train, auto_y_test = train_test_split(auto_X, auto_y, test_size=0.3, random_state=SEED) # Split the data into 70% train and 30% test
auto_dt = DecisionTreeRegressor(max_depth=4, min_samples_leaf=0.26, random_state=SEED) # Instantiate a DecisionTreeRegressor dt

print("****************************************************")
tema = "6. Evaluate the 10-fold CV error"; print("** %s\n" % tema)

auto_MSE_CV_scores = - cross_val_score(auto_dt, auto_X_train, auto_y_train, cv=10, scoring='neg_mean_squared_error', n_jobs=-1) # Compute the array containing the 10-folds CV MSEs
RMSE_CV = (auto_MSE_CV_scores.mean())**(1/2) # Compute the 10-folds CV RMSE

print('CV RMSE: {}'.format(RMSE_CV)) # Print RMSE_CV

print("****************************************************")
tema = "7. Evaluate the training error"; print("** %s\n" % tema)

auto_dt.fit(auto_X_train, auto_y_train) # Fit dt to the training set
auto_y_pred_train = auto_dt.predict(auto_X_train) # Predict the labels of the training set
auto_RMSE_train = (MSE(auto_y_train, auto_y_pred_train))**(1/2) # Evaluate the training set RMSE of dt

print('Train RMSE: {}'.format(auto_RMSE_train)) # Print RMSE_train

print("****************************************************")
tema = "8. High bias or high variance?"; print("** %s\n" % tema)

SEED = 3

auto_X_train, auto_X_test, auto_y_train, auto_y_test = train_test_split(auto_X, auto_y, test_size=0.2, random_state=SEED)
scalar = StandardScaler()

lr = LinearRegression()
pipeline = make_pipeline(scalar, lr)
pipeline.fit(auto_X_train, auto_y_train) # Fit dt to the training set
auto_y_pred = pipeline.predict(auto_X_test) # Compute y_pred
auto_mse_lr = MSE(auto_y_test, auto_y_pred) # Compute mse_dt
auto_rmse_lr = auto_mse_lr**(1/2) # Compute rmse_dt

print('Baseline RMSE: {}'.format(auto_rmse_lr)) # Print RMSE_train
print("This model suffers from high bias because RMSE_CV ≈ RMSE_train and both scores are greater than baseline_RMSE. \nIt's necessary to increase the complexity of the model")

print("****************************************************")
tema = "10. Define the ensemble"; print("** %s\n" % tema)

SEED=1 # Set seed for reproducibility

lr = LogisticRegression(solver='liblinear', random_state=SEED) # Instantiate lr
knn = KNN(n_neighbors=27) # Instantiate knn
dt = DecisionTreeClassifier(min_samples_leaf=0.13, random_state=SEED) # Instantiate dt
#lr = LogisticRegression(n_jobs=1, multi_class='ovr', solver='liblinear', random_state=SEED) # Instantiate lr
#knn = KNN(n_jobs=1, n_neighbors=27) # Instantiate knn
#dt = DecisionTreeClassifier(min_samples_leaf=0.13, random_state=SEED) # Instantiate dt

classifiers = [('Logistic Regression', lr), ('K Nearest Neighbours', knn), ('Classification Tree', dt)] # Define the list classifiers
print(lr.get_params)
print(knn.get_params)
print(dt.get_params)

print("****************************************************")
tema = "11. Evaluate individual classifiers"; print("** %s\n" % tema)

liver_X_train, liver_X_test, liver_y_train, liver_y_test = train_test_split(liver_X, liver_y, test_size=0.3, random_state=SEED)

for clf_name, clf in classifiers:                         # Iterate over the pre-defined list of classifiers
    clf.fit(liver_X_train, liver_y_train)                 # Fit clf to the training set
    liver_y_pred = clf.predict(liver_X_test)              # Predict y_pred
    accuracy = accuracy_score(liver_y_test, liver_y_pred) # Calculate accuracy
    print('{:s}: {:.6f}. Patients with a prediction of liver disease: {}.'.format(clf_name, accuracy, liver_y_pred.sum()))     # Evaluate clf's accuracy on the test set

print("****************************************************")
tema = "12. Better performance with a Voting Classifier"; print("** %s\n" % tema)

vc = VotingClassifier(estimators=classifiers)             # Instantiate a VotingClassifier vc
vc.fit(liver_X_train, liver_y_train)                      # Fit vc to the training set
liver_y_pred = vc.predict(liver_X_test)                   # Evaluate the test set predictions
accuracy = accuracy_score(liver_y_test, liver_y_pred)     # Calculate accuracy score

print('Voting Classifier: {:.6f}'.format(accuracy))

print("****************************************************")
print("** END                                            **")
print("****************************************************")

Machine Learning with Tree-Based Models in Python/02_Generalization_Error.py


from django.urls import path
from . import views
from django.conf.urls.static import static
from django.conf import settings

urlpatterns=[
    path('products/',views.ProductList.as_view()),
    path('products/<int:pk>/',views.ProductDetail.as_view()),
    path('variation/<int:pk>/',views.EditVariation.as_view()),
    path('image/',views.AddImage.as_view()),
    path('rate/',views.Ratings.as_view()),
    path('comment/',views.Comments.as_view()),
    path('category/',views.CategoryList.as_view()),
    path('category/<int:pk>/',views.CategoryDetail.as_view()),
    path('receiver/',views.ReceiverInformations.as_view()),
    path('receiver/<int:pk>/',views.ReceiverInformationDetail.as_view()),
    path('province/',views.province_choices),
    path('coupon/',views.Coupons.as_view()),
    path('coupon/<int:pk>/',views.CouponDetail.as_view()),
    path('profile/',views.Profiles.as_view()),
    # path('profile/me/',views.ProfilesMe.as_view()), need viewset action
    path('profile/me/',views.profile_me),
    path('profile/<int:pk>/',views.ProfileDetail.as_view()),
    path('basket/',views.Baskets.as_view()),# don't need maybe
    path('basket/<uuid:pk>/',views.BasketDetail.as_view()),
    path('items/',views.AddItems.as_view()),
    path('items/<int:pk>',views.EditRemoveItems.as_view()),

]
# + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
if settings.DEBUG:
    urlpatterns += static(settings.MEDIA_URL,document_root=settings.MEDIA_ROOT)


import socket
import requests

"""Using page 166 from chapter 2.7 of Computer Networking, A Top Down Approach, adapted TCPclient.py
to implement a simple GET request. Additional resources used for GET specific information from 'Socket programing
HOWTO' docs pages: https://docs.python.org/3/howto/sockets.html

Instructions:
Ensure socket, request, and json libraries are installed in local system  and imported.
Copy get_external_ip() and get_ip_loc() functions from ip_locator to user service.
Assigned a local variable to return get_ip_loc() to.
Run ip_locator_serv.py then start your app.
Can be used continuously, no need to restart the ip_locator_serv
"""

def get_external_ip():
    """
    Users likely using DHCP and local private address could cause issues for the location request.
    This function will return a user's externally facing public ip assigned in DHCP.
    """
    #API for external ip lookup: https://www.ipify.org/
    return requests.get('https://api.ipify.org').text

def get_ip_loc():
    HOST = '127.0.0.2'                                      #Adjust as necessary; match with ip_locator_serv
    PORT = 18777                                            #Adjust as necessary; match with ip_locator_serv

    #Setup socket to connect to local server
    user_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    #Use get_external_ip to get the user's current DHCP; if not using DHCP should still work as expected
    ip_request = get_external_ip()

    try:
        user_socket.connect((HOST, PORT))
        user_socket.sendall(ip_request.encode())
        ip_location = user_socket.recv(1024).decode()

        return ip_location

    except Exception as e:
        print("Cannot connect to the server:", e)

#Assign returned location json data to a variable
user_location = get_ip_loc()
print(user_location)