from Crypto.Cipher import AES
import bson
from bson.binary import Binary, UUIDLegacy
from pymongo import config

key = config.key

obj = AES.new(key)

def encrypt_helper(s):
	length = 16 - (len(s) % 16)
	s += chr(length) * length
	s = obj.encrypt(s)
	s = Binary(s, 1)
	return s

def decrypt_helper(s):
	s = obj.decrypt(s)
	buf = bytearray(s)
	length = buf[-1]
	s = s[: -length]
	return s

def encrypt_doc(doc):
	if isinstance(doc, str):
		return encrypt_helper(doc)
	if isinstance(doc, dict):
		for key in doc:
			if cmp(key, "channel") != 0:
				doc[key] = encrypt_doc(doc[key])
		return doc

	if isinstance(doc, list):
		for i in range(len(doc)):
			doc[i] = encrypt_doc(doc[i])
		return doc
	return doc

def decrypt_doc(doc):
	if isinstance(doc, Binary):
		return decrypt_helper(doc)
	if isinstance(doc, dict):
		temp_doc = {}
		for key in doc:
			#version 1
			#doc[key] = decrypt_doc(doc[key])

			#version 2
			temp_doc[str(key)] = decrypt_doc(doc[key])
		doc = temp_doc
		return doc

	if isinstance(doc, list):
		for i in range(len(doc)):
			doc[i] = decrypt_doc(doc[i])
		return doc
	return doc


#!/usr/bin/python3
import re
from os.path import exists
import sys
from subprocess import check_output

number_of_nmap = 0

def readfile(filename):
    f = open(filename,"r")
    data = f.readlines()
    return data

def parse_targets(data):
    # Creating a dict object with each port as the key. Ip addresses are key values
    targets = {}
    for d in data:
        if '#' not in d[0]:
            ip_address = []
            ip_address.append(d.split()[3])
            port = d.split()[2]
            if port not in targets:
                targets[port] = ip_address
            else:
                targets[port].extend(ip_address)
    return targets

def create_nmap_scans(targets):
    # We will generate an nmap scan command for each port with the required targets
    nmap_commands = []
    for port in targets:
        scan_targets = []
        for ip in targets[port]:
            scan_targets.append(ip)
        command = "nmap -sV -Pn -p" + str(port) + " " + ' '.join(scan_targets)
        nmap_commands.append(command)
        number_of_nmap = len(nmap_commands)
    print("%s total Nmap commands to be executed" % number_of_nmap)
    return nmap_commands

def run_nmap(command):
    #this runs an instance of nmap as a subprocess of this script
    results = check_output(command,shell=True)
    return results

def get_banner(results):
    # We now parse the resulting nmap stdout which has been captured to extract the specific values required
    banner_results = []
    regex_port='(\d\d?\d?\d?\d?)\/tcp'
    regex_banner = 'tcp\s(open|filtered).*'
    regex_host = '.*'
    results_array = results.decode("utf-8").split("Nmap scan report for ")

    for host in results_array[1:]:
        target = re.search(regex_host, host).group(0)
        port = re.search(regex_port, host).group(1)
        banner = ""
        #regex was acting quirky, so bare with me here...
        banner_array = re.search(regex_banner, host).group(0).split()
        length = len(banner)
        service = banner_array[2]
        count = 3
        while count < len(banner_array):
            banner = banner + " " + banner_array[count]
            count = count +1

        results = str(target) + "," + str(port) + "," + service + "," + str(banner)
        banner_results.append(results)
    return banner_results

def output_to_csv(banner,filename):
    # now we just print to the screen and $profit
    newfile = filename.split('.')
    newfile = newfile[0] + ".csv"
    if not exists(newfile):
        print("creating output file: %s" % newfile)
        f = open(newfile,'w')
        f.write("HOST,PORT,SERVICE,BANNER\n")
    else:
        f = open(newfile,'a')
        for results in banner:
            f.write(results + "\n")
    f.close()
    return

if __name__ == "__main__":
    if len(sys.argv) > 1:
        filename = sys.argv[1]
        count = 1
        # Read the masscan data file
        data = readfile(filename)

        # parse those targets by port
        targets = parse_targets(data)

        # Generate nmap commands for the targets based on the port to be queried
        nmap_commands = create_nmap_scans(targets)
        number_of_nmap = len(nmap_commands)
        for command in nmap_commands:
            print("(%i of %i) executing: %s" % (count,number_of_nmap,command))
            #run the required nmap scans, one for each port
            results = run_nmap(command)
            # get the banner information
            banner = get_banner(results)
            #output the nmap results to the stdout
            output_to_csv(banner,filename)
            count = count + 1

    else:
        print("usage: %s: MASSCAN OUTPUT FILE" % sys.argv[0])
    exit()


# -*- coding: utf-8 -*-

from setuptools import setup, find_packages

tests_require = [
    'pytest==5.2.2',
    'pytest-aiohttp==0.3.0',
    'nose',
    'coverage',
    'flake8',
]

setup(
    name='aiohttp-prometheus',
    version='0.1.0',
    description="HTTP metrics for a aiohttp application",
    long_description=open('README.rst').read(),
    keywords='prometheus aiohttp',
    author=u'Globo.com',
    author_email='backstage@corp.globo.com',
    url='https://github.com/globocom/aiohttp-prometheus',
    license='MIT',
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Natural Language :: English',
        'Operating System :: MacOS',
        'Operating System :: POSIX',
        'Operating System :: Unix',
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.5',
    ],
    packages=find_packages(
        exclude=(
            'tests',
        ),
    ),
    include_package_data=True,
    install_requires=[
        'aiohttp',
        'prometheus-client==0.7.1',
    ],
    extras_require={
        'tests': tests_require,
    },
)


from sqlalchemy import Column, String, Integer
from sqlalchemy.orm import relationship
from sqlalchemy.orm import validates

from pydantic import validate_email

from base import Base

class Customer(Base):
    __tablename__ = 'customer'

    id = Column(Integer, primary_key=True)
    name = Column(String(50), nullable=False)
    email = Column(String(50), nullable=False, unique=True)
    mobile_number = Column(String(20), nullable=False, unique=True)
    # it doesn't matter what name you are giving as backref argument
    reviews = relationship('Review', backref='customer', cascade="all, delete, delete-orphan")

    @validates('mobile_number')
    def no_chars_in_mobile_no(self, key, v):
        if any(i.isalpha() for i in v):
            raise ValueError("Invalid Mobile Number")
        return v

    @validates('email')
    def check_email(self, key, v):
        if not validate_email(v):
            raise ValueError("Invalid Email")
        return v

    def __init__(self, name, email, mobile_number):
        self.name = name
        self.email = email
        self.mobile_number = mobile_number

    def __repr__(self):
        return "Customer({}, {}, {})".format(self.name, self.email, self.mobile_number)


import unittest

from unittest.mock import patch

from io import StringIO
from pathlib import Path
from typing import List

from ospd.parser import (
    DEFAULT_MQTT_BROKER_ADDRESS,
    DEFAULT_MQTT_BROKER_PORT,
    Arguments,
    DEFAULT_ADDRESS,
    DEFAULT_PORT,
    DEFAULT_KEY_FILE,
    DEFAULT_NICENESS,
    DEFAULT_SCANINFO_STORE_TIME,
    DEFAULT_UNIX_SOCKET_PATH,
    DEFAULT_PID_PATH,
    DEFAULT_LOCKFILE_DIR_PATH,
)
from ospd_openvas.notus import NotusParser

here = Path(__file__).absolute().parent

class ArgumentParserTestCase(unittest.TestCase):
    def setUp(self):
        self.parser = NotusParser()

    def parse_args(self, args: List[str]) -> Arguments:
        return self.parser.parse_arguments(args)

    @patch('sys.stderr', new_callable=StringIO)
    def test_port_interval(self, _mock_stderr):
        with self.assertRaises(SystemExit):
            self.parse_args(['--port=65536'])

        with self.assertRaises(SystemExit):
            self.parse_args(['--port=0'])

        args = self.parse_args(['--port=3353'])
        self.assertEqual(3353, args.port)

    @patch('sys.stderr', new_callable=StringIO)
    def test_port_as_string(self, _mock_stderr):
        with self.assertRaises(SystemExit):
            self.parse_args(['--port=abcd'])

    def test_address_param(self):
        args = self.parse_args('-b 1.2.3.4'.split())
        self.assertEqual('1.2.3.4', args.address)

    def test_correct_lower_case_log_level(self):
        args = self.parse_args('-L error'.split())
        self.assertEqual('ERROR', args.log_level)

    def test_correct_upper_case_log_level(self):
        args = self.parse_args('-L INFO'.split())
        self.assertEqual('INFO', args.log_level)

    @patch('sys.stderr', new_callable=StringIO)
    def test_correct_log_level(self, _mock_stderr):
        with self.assertRaises(SystemExit):
            self.parse_args('-L blah'.split())

    def test_non_existing_key(self):
        args = self.parse_args('-k foo'.split())
        self.assertEqual('foo', args.key_file)

    def test_existing_key(self):
        args = self.parse_args('-k /etc/passwd'.split())
        self.assertEqual('/etc/passwd', args.key_file)

    def test_disable_notus_hashsum_verification(self):
        args = self.parse_args(
            '--disable-notus-hashsum-verification true'.split()
        )
        self.assertEqual(args.disable_notus_hashsum_verification, True)

    def test_defaults(self):
        args = self.parse_args([])

        self.assertIsNone(args.config)
        self.assertEqual(args.key_file, DEFAULT_KEY_FILE)
        self.assertEqual(args.niceness, DEFAULT_NICENESS)
        self.assertEqual(args.log_level, 'INFO')
        self.assertEqual(args.address, DEFAULT_ADDRESS)
        self.assertEqual(args.port, DEFAULT_PORT)
        self.assertEqual(args.scaninfo_store_time, DEFAULT_SCANINFO_STORE_TIME)
        self.assertEqual(args.unix_socket, DEFAULT_UNIX_SOCKET_PATH)
        self.assertEqual(args.pid_file, DEFAULT_PID_PATH)
        self.assertEqual(args.lock_file_dir, DEFAULT_LOCKFILE_DIR_PATH)
        self.assertEqual(args.mqtt_broker_address, DEFAULT_MQTT_BROKER_ADDRESS)
        self.assertEqual(args.mqtt_broker_port, DEFAULT_MQTT_BROKER_PORT)
        self.assertEqual(args.disable_notus_hashsum_verification, False)

class ArgumentParserConfigTestCase(unittest.TestCase):
    def setUp(self):
        self.parser = NotusParser()

    def parse_args(self, args: List[str]) -> Arguments:
        return self.parser.parse_arguments(args)

    def test_using_config(self):
        config_file = str(here / 'testing.conf')
        args = self.parse_args(['--config', config_file])

        self.assertEqual(args.key_file, '/foo/key.pem')
        self.assertEqual(args.niceness, 666)
        self.assertEqual(args.log_level, 'DEBUG')
        self.assertEqual(args.address, '6.6.6.6')
        self.assertEqual(args.port, 6666)
        self.assertEqual(args.scaninfo_store_time, 123)
        self.assertEqual(args.config, config_file)
        self.assertEqual(args.unix_socket, '/foo/ospd-openvas.sock')
        self.assertEqual(args.pid_file, '/foo/ospd-openvas.pid')
        self.assertEqual(args.lock_file_dir, '/foo/openvas')
        self.assertEqual(args.mqtt_broker_address, 'foo.bar.com')
        self.assertEqual(args.mqtt_broker_port, 1234)
        self.assertEqual(args.notus_feed_dir, '/foo/advisories')

    @patch('sys.stderr', new_callable=StringIO)
    def test_not_existing_config(self, _mock):
        with self.assertRaises(SystemExit):
            self.parse_args(['--config', 'foo.conf'])


#
# https://github.com/andymccurdy/redis-py
#
import time
import configparser
import redis

def read_host_name():
    config = configparser.ConfigParser()
    config['General'] = {'host': 'localhost'}
    config.read('user.ini')
    host = config['General']['host']
    return host

def connect_to_redis(host):
    redis_connection = redis.StrictRedis(host=host, port=6379, db=0)
    return redis_connection

def set_get_delete(redis_connection):
    print('set_get_delete <<<<<<<<<<<<<<<<<<<<<<')
    simple_key = 'igor:simple'
    result = redis_connection.set(simple_key, 'intro')
    print('set: ', result)

    result = redis_connection.get(simple_key)
    print('get: ', result)

    redis_connection.delete(simple_key)
    print('get (after delete): ', redis_connection.get(simple_key))
    print('set_get_delete >>>>>>>>>>>>>>>>>>>>>>')
    return

def pub_sub_1(redis_connection):
    print('pub_sub1 <<<<<<<<<<<<<<<<<<<<<<')
    channel_key = 'igor:my-1-channel'
    p = redis_connection.pubsub()
    p.subscribe(channel_key)
    time.sleep(2)
    msg = p.get_message()
    print('get_message', msg)

    number_of_subscribers = redis_connection.publish(channel_key, 'some data')
    print('number_of_subscribers: ', number_of_subscribers)
    msg = p.get_message()
    print('get_message', msg)

    p.unsubscribe(channel_key)
    print('pub_sub1 >>>>>>>>>>>>>>>>>>>>>>')
    return

def my_handler(message):
    print('MY HANDLER: ', message['data'])
    return

def pub_sub_2(redis_connection):
    print('pub_sub2 <<<<<<<<<<<<<<<<<<<<<<')
    channel_key = 'igor:my-2-channel'
    p = redis_connection.pubsub()
    p.subscribe(**{channel_key: my_handler})
    time.sleep(2)
    msg = p.get_message()
    print('get_message', msg)

    number_of_subscribers = redis_connection.publish(channel_key, 'awesome data')
    print('number_of_subscribers: ', number_of_subscribers)
    msg = p.get_message()
    print('get_message', msg)

    p.unsubscribe(channel_key)
    print('pub_sub2 >>>>>>>>>>>>>>>>>>>>>>')
    return

def pub_sub_3(redis_connection):
    print('pub_sub3 <<<<<<<<<<<<<<<<<<<<<<')
    channel_key = 'igor:my-3-channel'
    p = redis_connection.pubsub(ignore_subscribe_messages=True)
    p.subscribe(channel_key)

    for message in p.listen():
        print('get_message', message)
        if message is not None and message['data'] == b'finish':
            break

    p.unsubscribe(channel_key)
    print('pub_sub3 >>>>>>>>>>>>>>>>>>>>>>')
    return

if __name__ == "__main__":
    print("main started")
    host = read_host_name()
    print('host:', host)

    redis_connection = connect_to_redis(host)

    set_get_delete(redis_connection)

    pub_sub_1(redis_connection)
    pub_sub_2(redis_connection)
    pub_sub_3(redis_connection)

    print("main completed")


"""Biweekly Contest 26 - Q1"""

class Solution:
    """my sol - first attempt"""
    def maxPower(self, s: str) -> int:

        cur = 1
        power = 1

        for i in range(1, len(s)):

            if s[i] == s[i - 1]:
                cur += 1
            else:
                cur = 1

            power = max(power, cur)

        return power

    """online sol - 1 line"""
    def maxPower(self, s: str) -> int:

        return max(len(list(b)) for a, b in itertools.groupby(s))


#!/usr/bin/env python
import rospy

from std_msgs.msg import Int16
from project1_solution.msg import TwoInts

def callback(data):
    rospy.loginfo(rospy.get_caller_id() + 'I heard %s', data)

    pub = rospy.Publisher('sum', Int16, queue_size = 10)

    rate = rospy.Rate(0.5)

    #while not rospy.is_shutdown():
    final_sum = data.a + data.b
    rospy.loginfo('sum = ' + str(final_sum))
    pub.publish(final_sum)
    rate.sleep()

def solution():
    rospy.init_node('solution', anonymous = True)

    #Subscribe to two_ints topic and publishes to sum topic
    rospy.Subscriber('two_ints', TwoInts, callback)

    rospy.spin()

if __name__ == '__main__':
    try:
        solution()
    except rospy.ROSInterruptException:
        pass


import unittest

from app import create_app
from shopyoapi.init import db

class BaseTest(unittest.TestCase):
    """Base test class"""

    def setUp(self):
        """Define test variables and initialize app."""
        self.app = create_app(config_name="testing")
        self.client = self.app.test_client

        with self.app.app_context():
            # create all tables
            db.create_all()

    def tearDown(self):
        """teardown all initialized variables."""
        with self.app.app_context():
            # drop all tables
            db.session.remove()
            db.drop_all()


def create_table(name_table, cur):
    query = """
     CREATE TABLE IF NOT EXISTS {} (
        dimension TEXT PRIMARY KEY NOT NULL,
        probability VARCHAR(30) NOT NULL)""".format(name_table)

    cur.execute(query)
    cur.execute("DELETE FROM {}".format(name_table))

def create_size(con, length_x):
    cur = con.cursor()
    query="""
    CREATE TABLE IF NOT EXISTS length_table (
        length_t INTEGER PRIMARY KEY NOT NULL DEFAULT '{}'
        )""".format(length_x)
    cur.execute(query)
    cur.execute("INSERT OR IGNORE INTO length_table (length_t) VALUES (?)",(length_x,))
    con.commit()

def create_table_hash( cur):
    query = """
     CREATE TABLE IF NOT EXISTS {} (
        hash_t VARCHAR(30) NOT NULL)""".format("hash_table")

    cur.execute(query)
    #cur.execute("DELETE FROM {}".format(name_table))


import pandas as pd
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
import numpy as np
from mpl_toolkits.mplot3d import Axes3D

def test():
    arr = np.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]])
    print(arr)
    arr=np.delete(arr, -1, axis=1)
    print(arr)
    input()

def PCA2D(X,tag):
    pca=PCA(n_components=2)
    new_X=pca.transform(X)
    x,y=[],[],[]
    for i in range(len(new_X)):
        x.append(new_X[i][0])
        y.append(new_X[i][1])
    plt.scatter(x,y,c=tag)
    plt.show()
def PCA3D(X,tag):
    pca=PCA(n_components=3)
    new_X=pca.transform(X)
    fig = plt.figure()
    ax=fig.add_subplot(projection='3d')
    ax.scatter(new_X[:, 0], new_X[:, 1], new_X[:, 2],c=tag)
    plt.show()
def preHandleData(X):
    X = np.delete(X, -1, axis=1)

if __name__=='__main__':
    csv_data=pd.read_csv('./data/KDD99_train_set_origin.csv')
    print(csv_data.shape)
    X=np.array(csv_data.values)
    tag=X[:,41]
    X=np.delete(X,-1,axis=1)

    PCA2D(X,tag)


#!/usr/bin/env python
# coding: utf-8

# # Traducción de texto mediante algoritmos de Machine learning

# ### Importación de las librerías

# In[1]:

import string
import re
from numpy import array, argmax, random, take
import pandas as pd
from keras.models import Sequential
from keras.layers import Dense, LSTM, Embedding, Bidirectional, RepeatVector, GRU, SimpleRNN
from keras.layers.embeddings import Embedding
from keras.preprocessing.text import Tokenizer
from keras.callbacks import ModelCheckpoint
from keras.preprocessing.sequence import pad_sequences
from keras.models import load_model
from keras import optimizers
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
pd.set_option('display.max_colwidth', 200)

# ### Lectura de datos

# Los datos son un archivo de texto de pares de oraciones en inglés-español.

# In[2]:

# Leemos el archivo
def read_text(filename):
    # open the file
    file = open(filename, mode='rt', encoding='utf-8')
    # read all text
    text = file.read()
    file.close()
    return text

# Ahora definamos una función para dividir el texto en pares inglés-español separados por '\ n'. Luego dividimos estos pares en oraciones de inglés y español.

# In[3]:

# Dividimos en frases
def to_lines(text):
    sents = text.strip().split('\n')
    sents = [i.split('\t') for i in sents]
    return sents

# Para descargar los datos pulse [aquí.](http://www.manythings.org/anki/deu-eng.zip)__ y extrae "spa.txt" en tu directorio de trabajo.

# In[4]:

data = read_text("C:/Users/María/MASTER/Datos No Estruct/Texto/Practica/data/spa.txt")
spa_eng = to_lines(data)
spa_eng = array(spa_eng)

# Los datos contienen más de 150 mil pares de oraciones. Sin embargo, usaremos los primeros 50 mil pares de oraciones para reducir el tiempo de entrenamiento del modelo.

# In[5]:

spa_eng = spa_eng[:50000,:]

# In[6]:

spa_eng.shape

# ### Limpieza de texto

# Echemos un vistazo a nuestros datos:

# In[7]:

spa_eng

# Eliminaremos los signos de puntuación y luego convertiremos el texto a minúsculas.

# In[8]:

# Eliminamos los signos de puntuación
spa_eng[:,0] = [s.translate(str.maketrans('', '', string.punctuation)) for s in spa_eng[:,0]]
spa_eng[:,1] = [s.translate(str.maketrans('', '', string.punctuation)) for s in spa_eng[:,1]]

# In[9]:

spa_eng

# In[10]:

# Convertimos el texto a minúsculas
for i in range(len(spa_eng)):
    spa_eng[i,0] = spa_eng[i,0].lower()

    spa_eng[i,1] = spa_eng[i,1].lower()

# In[11]:

spa_eng

# ### Preprocesamiento de texto

# - #### Conversión de texto a secuencia

# Para alimentar nuestros datos en un modelo *Seq2Seq*, tendremos que convertir las oraciones de entrada y de salida en secuencias enteras de longitud fija. Antes de eso, visualizamos la longitud de las oraciones en dos listas separadas para inglés y para español, respectivamente.

# In[12]:

# Listas vacias
eng_l = []
spa_l = []

# Rellenar las listas con longitudes de oraciones
for i in spa_eng[:,0]:
    eng_l.append(len(i.split()))

for i in spa_eng[:,1]:
    spa_l.append(len(i.split()))

# In[13]:

length_df = pd.DataFrame({'Inglés':eng_l, 'Español':spa_l})
print(length_df)
print(max(length_df.Inglés))
print(max(length_df.Español))

# In[14]:

length_df.hist(bins = 30)
plt.show()

# La longitud máxima de las frases en Español es de 10 y la de las frases en Inglés es de 7.

# - #### Tokenización

# A continuación, necesitamos tokenizar los datos, es decir, convertir el texto en valores numéricos. Esto permite que la red neuronal realice operaciones sobre los datos de entrada.

# Vectoricemos nuestros datos de texto usando la clase *Tokenizer ()* de Keras. Convertirá nuestras oraciones en secuencias de números enteros. Luego rellenaremos esas secuencias con ceros para hacer que todas las secuencias tengan la misma longitud.

# In[15]:

# Función para construir un tokenizador
def tokenization(lines):
    tokenizer = Tokenizer()
    tokenizer.fit_on_texts(lines)
    return tokenizer

# In[16]:

# Preparar tokenizador en inglés
eng_tokenizer = tokenization(spa_eng[:, 0])
eng_vocab_size = len(eng_tokenizer.word_index) + 1

eng_length = 8
print('Tamaño del vocabulario en inglés: %d' % eng_vocab_size)

# In[17]:

# Preparar tokenizador en español
spa_tokenizer = tokenization(spa_eng[:, 1])
spa_vocab_size = len(spa_tokenizer.word_index) + 1

spa_length = 8
print('Tamaño del vocabulario en español: %d' % spa_vocab_size)

# In[18]:

print(spa_tokenizer.word_counts)

# A continuación se muestra una función para preparar las secuencias. También realizará el relleno de secuencia hasta su longitud máxima.

# Cuando introducimos nuestras secuencias de ID de palabras en el modelo, cada secuencia debe tener la misma longitud. Para lograr esto, se agrega relleno a cualquier secuencia que sea más corta que la longitud máxima (es decir, más corta que la oración más larga).

# In[19]:

# Codificar y rellenar secuencias
def encode_sequences(tokenizer, length, lines):
    # Secuencias de codificación entera
    seq = tokenizer.texts_to_sequences(lines)
    # Rellenar las secuencias con 0
    seq = pad_sequences(seq, maxlen=length, padding='post')
    print(seq)
    print(len(seq))
    return seq

# eng_padding= encode_sequences(eng_tokenizer, 12, spa_eng[:, 0])

# ### Creación de la muestra de train y test

# Dividimos los datos para el entrenamiento y la evaluación del modelo.

# In[20]:

from sklearn.model_selection import train_test_split
train, test = train_test_split(spa_eng, test_size=0.2, random_state = 12)

# Codificamos las oraciones de español como secuencias de entrada y las oraciones de inglés como secuencias de destino. Para el conjunto de train y test.

# In[21]:

# Preparación de los datos de entrenamiento
trainX = encode_sequences(spa_tokenizer, spa_length, train[:, 1])
trainY = encode_sequences(eng_tokenizer, eng_length, train[:, 0])

# In[22]:

print(trainX.shape)
print(trainY.shape)

# In[23]:

# Preparación de los datos de test
testX = encode_sequences(spa_tokenizer, spa_length, test[:, 1])
testY = encode_sequences(eng_tokenizer, eng_length, test[:, 0])

# In[24]:

print(testX.shape)
print(testY.shape)

# ### Construcción del modelo

# Partes de la arquitectura de un RNN en un nivel alto:
#
# - **Entradas.** Las secuencias de entrada se introducen en el modelo con una palabra para cada paso de tiempo. Cada palabra está codificada como un número entero único o un vector codificado en caliente que se asigna al vocabulario del conjunto de datos en inglés.
#
#
# - **Embeddings.** Las incrustaciones se utilizan para convertir cada palabra en un vector. El tamaño del vector depende de la complejidad del vocabulario.
#
#
# - **Capas recurrentes (codificador).** Aquí es donde el contexto de los vectores de palabras en los pasos de tiempo anteriores se aplica al vector de palabras actual.
#
#
# - **Capas densas (decodificador).** Estas son capas típicas completamente conectadas que se utilizan para decodificar la entrada codificada en la secuencia de traducción correcta.
#
#
# - **Salidas.** Las salidas se devuelven como una secuencia de números enteros o vectores codificados en caliente que luego se pueden asignar al vocabulario del conjunto de datos.

# Los **Embeddings** permiten capturar relaciones de palabras sintácticas y semánticas más precisas proyectando cada palabra en un espacio n-dimensional. Las palabras con significados similares ocupan regiones similares de este espacio; cuanto más cercanas están dos palabras, más similares son. Y a menudo los vectores entre palabras representan relaciones útiles, como género, tiempo verbal o incluso relaciones geopolíticas.
#
# El entrenamiento con Embeddings en un gran conjunto de datos desde cero requiere una gran cantidad de datos y cálculos. Entonces, en lugar de hacerlo nosotros mismos, normalmente usamos un paquete de Embeddings previamente entrenado como word2vec . Cuando se usan de esta manera, las incrustaciones son una forma de aprendizaje por transferencia. Sin embargo, dado que nuestro conjunto de datos para este proyecto tiene un vocabulario pequeño y poca variación sintáctica, usaremos Keras para entrenar las incrustaciones nosotros mismos.

# Nuestro **modelo secuencia a secuencia** vincula dos redes recurrentes: un codificador y un descodificador. El codificador resume la entrada en una variable de contexto. Depués, se decodifica este contexto y se genera la secuencia de salida. Dado que tanto el codificador como el descodificador son recurrentes, tienen bucles que procesan cada parte de la secuencia en diferentes pasos de tiempo.
#
# Para cada paso de tiempo después de la primera palabra en la secuencia hay dos entradas: el estado oculto y una palabra de la secuencia. Para el codificador, es la siguiente palabra en la secuencia de entrada. Para el decodificador, es la palabra anterior de la secuencia de salida.
#
# El aprendizaje secuencia a secuencia (Seq2Seq) se trata de entrenar modelos para convertir secuencias de un dominio (inglés) a secuencias en otro dominio (español). Es útil para la generación de texto como traducción automática y respuesta a preguntas. Definimos la arquitectura *Seq2Seq*. Estamos usando una capa de incrustación y una capa LSTM como codificador y otra capa LSTM seguida de una capa densa como decodificador.

# Ya entendemos cómo fluye el contexto a través de la red a través del estado oculto, elsiguiente paso es entender como fluye el contexto en ambas direcciones. Esto es lo que hace una **capa bidireccional**. Proporcionar un contexto futuro puede resultar en un mejor rendimiento del modelo.
#
# Esto puede parecer contrario a la forma en que los humanos procesan el lenguaje, ya que solo leemos en una dirección. Sin embargo, los humanos a menudo requieren un contexto futuro para interpretar lo que se dice. En otras palabras, a veces no entendemos una oración hasta que se proporciona una palabra o frase importante al final. Para implementar esto, entrenamos dos capas RNN simultáneamente. La primera capa se alimenta con la secuencia de entrada tal cual y la segunda se alimenta con una copia invertida.

# In[25]:

# Modelo LSTM
def build_model(in_vocab, out_vocab, in_timesteps, out_timesteps, units):
    model = Sequential()
    model.add(Embedding(in_vocab, units, input_length=in_timesteps, mask_zero=True))
    model.add(LSTM(units))
    model.add(RepeatVector(out_timesteps))
    model.add(LSTM(units, return_sequences=True))
    model.add(Dense(out_vocab, activation='softmax'))

    return model

# In[26]:

# Modelo RNN
def build_model_rnn(in_vocab, out_vocab, in_timesteps, out_timesteps, units):
    model = Sequential()
    model.add(Embedding(in_vocab, units, input_length=in_timesteps, mask_zero=True))
    model.add(SimpleRNN(units))
    model.add(RepeatVector(out_timesteps))
    model.add(SimpleRNN(units, return_sequences=True))
    model.add(Dense(out_vocab, activation='softmax'))

    return model

# In[27]:

# Modelo GRU
def build_model_gru(in_vocab, out_vocab, in_timesteps, out_timesteps, units):
    model = Sequential()
    model.add(Embedding(in_vocab, units, input_length=in_timesteps, mask_zero=True))
    model.add(GRU(units, return_sequences=True))
    model.add(SimpleRNN(units))
    model.add(RepeatVector(out_timesteps))
    model.add(Dense(out_vocab, activation='softmax'))

    return model

# Usamos el optimizador __'RMSprop (Root Mean Square Propagation)'__ ya que suele ser una buena opción para redes neuronales recurrentes. Otros tipos de optimizadores de redes neuronales profundas: Stochastic Gradient Descent (SGD), Adaptive Gradient Algorithm (AdaGrad), Adam (Adaptive moment estimation)...

# Como función de pérdida hemos utilizado __'sparse_categorical_crossentropy'__ porque nos permite usar la secuencia de destino tal y como está, en lugar de un formato codificado en caliente. Una codificación en caliente de las secuencias de destino con un vocabulario tan grande podría consumir toda la memoria de nuestro sistema. Otros tipos de funciones de pérdida: CategoricalCrossentropy, BinaryCrossentropy.

# Podemos comenzar a entrenar nuestro modelo. Lo entrenaremos durante 30 épocas, con un tamaño de lote de 512. También usaremos __ModelCheckpoint ()__ para guardar el mejor modelo con la menor pérdida de test. Este método sirve para evitar la parada anticipada.

# In[28]:

model = build_model(spa_vocab_size, eng_vocab_size, spa_length, eng_length, 512)
rms =   optimizers.RMSprop(lr=0.001)
model.compile(optimizer=rms, loss='sparse_categorical_crossentropy')

# In[29]:

model2 = build_model_rnn(spa_vocab_size, eng_vocab_size, spa_length, eng_length, 512)
rms =    optimizers.RMSprop(lr=0.001)
model2.compile(optimizer=rms, loss='sparse_categorical_crossentropy')

# In[30]:

model3 = build_model_gru(spa_vocab_size, eng_vocab_size, spa_length, eng_length, 512)
rms =    optimizers.RMSprop(lr=0.001)                                       # Defining the optimizing function
model3.compile(optimizer=rms, loss='sparse_categorical_crossentropy')       # Configuring the model for training

# A veces es útil apilar varias capas recurrentes una tras otra para aumentar el poder de representación de una red. Se deben obtener todas las capas intermedias para devolver secuencias completas.

# Más detalles sobre RepeatVector: https://machinelearningmastery.com/encoder-decoder-long-short-term-memory-networks/ Si se intenta forzar estas piezas del codificador-decodificador juntas, seobtiene un error que indica que la salida del decodificador es 2D y se requiere entrada 3D para el decodificador.
#
# Podemos resolver esto usando una capa *RepeatVector* que simplemente repite la entrada 2D proporcionada varias veces para crear una salida 3D. La capa *RepeatVector* se puede utilizar como un adaptador para ajustar el codificador y el decodificador de la red juntos. Podemos configurarlo para que repita el vector de longitud fija en cada paso de tiempo en la secuencia de salida.
#
# En resumen, el *RepeatVector* se utiliza como un **adaptador para ajustar la salida 2D de tamaño fijo del codificador a las diferentes longitudes y entradas 3D esperadas por el decodificador**. El contenedor *TimeDistributed* permite reutilizar la misma capa de salida para cada elemento en la secuencia de salida.

# In[ ]:

# DO NOT RUN IT (example of RepeatVector)
model = Sequential()
model.add(Dense(32, input_dim=32))

# now: model.output_shape == (None, 32)
# note: `None` is the batch dimension
model.add(RepeatVector(3))

# now: model.output_shape == (None, 3, 32)

# En el ejemplo anterior, la capa *RepeatVector* repite las entradas entrantes un número específico de tiempo. La forma de la entrada anterior era (32,). Pero la forma de salida del *RepeatVector* fue (3, 32), ya que las entradas se repitieron 3 veces.

# In[31]:

print(spa_vocab_size)
print(eng_vocab_size)
print(spa_length)
print(eng_length)

# - **LSTM**

# La capa SimpleRNN no es bueno al procesar secuencias largas, como texto. Esto se debe al problema del gradiente de desaparición. Otros tipos de capas recurrentes funcionan mucho mejor como LSTM Y GRU.
#
# **Long Short Term Memory o memoria a corto plazo (LSTM)** es una variante de la capa SimpleRNN que ya lleva información en muchos pasos de tiempo. Guarda la información para más tarde, evitando así que las señales más antiguas desaparezcan durante el procesamiento. Las unidades de memoria en un LSTM aprenden qué recordar y además pueden memorizar datos anteriores fácilmente.

# In[35]:

filename = 'model.h1.LSTM'
checkpoint = ModelCheckpoint(filename, monitor='val_loss', verbose=1, save_best_only=True, mode='min')

history = model.fit(trainX, trainY.reshape(trainY.shape[0], trainY.shape[1], 1),
          epochs=30,
          batch_size=512,
          validation_split = 0.2,
          callbacks=[checkpoint], verbose=1)

# In[29]:

model.summary()

# Comparamos la función de pérdida de train y test.

# In[30]:

plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.legend(['train','validation'])
plt.show()

# - **RNN**

# **Recurrent Neural Networks o redes neuronales recurrentes (RNN)** son un tipo de red neuronal donde la salida del paso anterior se alimenta como entrada al paso actual. El estado oculto  recuerda alguna información sobre una secuencia. Los RNN tienen una “memoria” que recuerda TODA la información sobre lo calculado. Utiliza los mismos parámetros para cada entrada, ya que realiza la misma tarea en todas las entradas para producir la salida. Esto reduce la complejidad de los parámetros.

# In[28]:

filename = 'model.h1.RNN'
checkpoint = ModelCheckpoint(filename, monitor='val_loss', verbose=1, save_best_only=True, mode='min')

# model2.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
history2 = model2.fit(trainX, trainY.reshape(trainY.shape[0], trainY.shape[1], 1),
                     epochs=20,
                     batch_size=128,
                     validation_split=0.2)

# In[29]:

model2.summary()

# In[30]:

plt.plot(history2.history['loss'])
plt.plot(history2.history['val_loss'])
plt.legend(['train','validation'])
plt.show()

# - **GRU**

# **Gated recurrent units o unidades recurrentes cerradas (GRUs)** son un mecanismo de compuerta en las redes neuronales recurrentes. El GRU es como una memoria a corto plazo (LSTM) PERO con  menos parámetros. LSTM tiene tres puertas (entrada, salida y puerta de olvido) y GRU tiene dos puertas (puerta de reinicio y actualización).
#
# La puerta de actualización (z) ayuda al modelo a determinar cuánta información de los pasos de tiempo anteriores debe transmitirse al futuro. Mientras tanto, la puerta de reinicio (r) decide cuánta información pasada se debe olvidar. Hace que las RNN sean un poco más inteligentes. En lugar de permitir que toda la información del estado oculto fluya a través de la red, tenemos que ser más selectivos con la información sea más relevante.
#
# LSTM es más preciso en los conjuntos de datos que utilizan una secuencia más larga pero GRU usa menos parámetros de entrenamiento y por lo tanto usa menos memoria; se ejecuta más rápido que LSTM. En resumen, si la secuencia es grande o la precisión es muy crítica,es mejor usar LSTM pero si queremos una operación más rápida y que use menos memoria optaremos GRU.

# In[28]:

filename = 'model.h1.GRU'
checkpoint = ModelCheckpoint(filename, monitor='val_loss', verbose=1, save_best_only=True, mode='min')

history3 = model3.fit(trainX, trainY.reshape(trainY.shape[0], trainY.shape[1], 1),
          epochs=30,
          batch_size=512,
          validation_split = 0.2,
          callbacks=[checkpoint], verbose=1)

# In[29]:

model3.summary()

# In[32]:

plt.plot(history3.history['loss'])
plt.plot(history3.history['val_loss'])
plt.legend(['train','validation'])
plt.show()

# Observando las funciones de pérdida de los tres modelos vemos que LSTM y GRU se sobre ajustan. Esto es algo normal ya que la traducción al final es muy dependiente del texto en el que está. Se comportan de forma muy parecida.
#
# LSTM (memoria a largo plazo) tiene tres puertas (entrada, salida y puerta de olvido), mientras que la GRU tiene dos puertas (puerta de reinicio y puerta de actualización). Las GRU asocian puertas de olvido y de entrada. Las GRU utilizan menos parámetros de entrenamiento y, por lo tanto, utilizan menos memoria, se ejecutan más rápidamente y se entrenan más rápido que las LSTM, mientras que éstas son más precisas en conjuntosde datos que utilizan una secuencia más larga (LSTM 80 min Y GRU 60 min).
#
# En resumen, si la secuencia es grande o la precisión es muy importante, se empleanLSTMs, mientras que para un menor consumo de memoria y un funcionamiento más rápido se escogen GRUs. Cuanto menor sea la función de pérdida; mejor será el modelo. Por lo que el mejor modelo es LSTM ya se empieza a estabilizar antes.

# ### Predicciones LSTM

# Cargamos el modelo guardado para hacer las predicciones.

# In[31]:

model = load_model('C:/Users/María/MASTER/Datos No Estruct/Texto/Practica/model.h1.LSTM')
preds = model.predict_classes(testX.reshape((testX.shape[0],testX.shape[1])))

# In[32]:

def get_word(n, tokenizer):
    for word, index in tokenizer.word_index.items():
        if index == n:
            return word
    return None

# In[33]:

# Convertir las predicciones en texto (inglés)
preds_text = []
for i in preds:
    temp = []
    for j in range(len(i)):
        t = get_word(i[j], eng_tokenizer)
        if j > 0:
            if (t == get_word(i[j-1], eng_tokenizer)) or (t == None):
                temp.append('')
            else:
                temp.append(t)

        else:
            if(t == None):
                temp.append('')
            else:
                temp.append(t)

    preds_text.append(' '.join(temp))

# In[34]:

pred_df = pd.DataFrame({'actual' : test[:,0], 'predicted' : preds_text})
pd.set_option('display.max_colwidth', 200)

# In[35]:

pred_df.head(15)

# In[36]:

pred_df.tail(15)

# In[37]:

pred_df.sample(15)

# ### Predicciones RNN

# In[31]:

model2 = load_model('C:/Users/María/MASTER/Datos No Estruct/Texto/Practica/model.h1.RNN')
preds = model2.predict_classes(testX.reshape((testX.shape[0],testX.shape[1])))

# In[32]:

def get_word(n, tokenizer):
    for word, index in tokenizer.word_index.items():
        if index == n:
            return word
    return None

# In[33]:

# Convertir las predicciones en texto (inglés)
preds_text = []
for i in preds:
    temp = []
    for j in range(len(i)):
        t = get_word(i[j], eng_tokenizer)
        if j > 0:
            if (t == get_word(i[j-1], eng_tokenizer)) or (t == None):
                temp.append('')
            else:
                temp.append(t)

        else:
            if(t == None):
                temp.append('')
            else:
                temp.append(t)

    preds_text.append(' '.join(temp))

# In[34]:

pred_df = pd.DataFrame({'actual' : test[:,0], 'predicted' : preds_text})
pd.set_option('display.max_colwidth', 200)

# In[35]:

pred_df.head(15)

# In[36]:

pred_df.tail(15)

# In[37]:

pred_df.sample(15)

# ### Predicciones GRU

# In[33]:

model3 = load_model('C:/Users/María/MASTER/Datos No Estruct/Texto/Practica/model.h1.GRU')
preds = model3.predict_classes(testX.reshape((testX.shape[0],testX.shape[1])))

# In[34]:

def get_word(n, tokenizer):
    for word, index in tokenizer.word_index.items():
        if index == n:
            return word
    return None

# In[35]:

# Convertir las predicciones en texto (inglés)
preds_text = []
for i in preds:
    temp = []
    for j in range(len(i)):
        t = get_word(i[j], eng_tokenizer)
        if j > 0:
            if (t == get_word(i[j-1], eng_tokenizer)) or (t == None):
                temp.append('')
            else:
                temp.append(t)

        else:
            if(t == None):
                temp.append('')
            else:
                temp.append(t)

    preds_text.append(' '.join(temp))

# In[36]:

pred_df = pd.DataFrame({'actual' : test[:,0], 'predicted' : preds_text})
pd.set_option('display.max_colwidth', 200)

# In[37]:

pred_df.head(15)

# In[38]:

pred_df.tail(15)

# In[39]:

pred_df.sample(15)

# Si comparamos a fondo las traducciones de los modelos; las traducciones de LSTM y GRU ya no se parecen tanto.


class Student:
    def __init__(self, name, surname, gender):
        self.name = name
        self.surname = surname
        self.gender = gender
        self.finished_courses = []
        self.courses_in_progress = []
        self.grades = {}
        self.average_rating = 0

    def add_courses(self, course_name):
        self.finished_courses.append(course_name)
# задание 3
    def __str__(self):
        a = []
        b = 0
        for key in self.grades.values():
            for i in key:
                b += i
                a.append(i)
        self.average_rating = b / len(a)
        res = f'Имя: {self.name}\nФамилия: {self.surname}\nСредняя оценка за домашние задания: {round(self.average_rating, 2)}\nКурсы в процессе изучения: {", ".join(self.courses_in_progress)}\nЗавершенные курсы: {", ".join(self.finished_courses)}'
        return res

#задание 2
    def rate_lector(self, lector, course, grade):
        if isinstance(lector, Lecturer) and course in lector.courses_attached and course in lector.courses_attached:
            if course in lector.grades:
                lector.grades[course] += [grade]
            else:
                lector.grades[course] = [grade]
        else:
            return 'Ошибка'

student_list = ["student_1", "student_2"]

# def student_average_rating(student_list, course):
#     for name in student_list:
#         print(getattr(name, 'grades'))

class Mentor:
    def __init__(self, name, surname):
        self.name = name
        self.surname = surname
        self.courses_attached = []
        self.average_rating = 0
        self.student_list = []

class Lecturer(Mentor):
    grades = {}

    def __str__(self):
        a = []
        b = 0
        for key in self.grades.values():
            for i in key:
                b += i
                a.append(i)
        self.average_rating = b / len(a)
        res = f'Имя: {self.name}\nФамилия: {self.surname}\nСредняя оценка за лекции: {round(self.average_rating, 2)}\n'
        return res
    def __lt__(self, other):
        if not isinstance(other, Student):
            print('Not a Student!')
        return self.average_rating < other.average_rating

class Rewiewer(Mentor):
# Задание 2
    def rate_hw(self, student, course, grade):
        if isinstance(student, Student) and course in self.courses_attached and course in student.courses_in_progress:
            if course in student.grades:
                student.grades[course] += [grade]
            else:
                student.grades[course] = [grade]
        else:
            return 'Ошибка'
    def __str__(self):
        res = f'Имя: {self.name}\nФамилия: {self.surname}\n'
        return res

# best_student = Student('Ruoy', 'Eman', 'your_gender')
# best_student.courses_in_progress += ['Python', 'GIT']
#
# cool_Rewiewer = Rewiewer('Some', 'Buddy')
# cool_Rewiewer.courses_attached += ['Python']
#
# cool_Rewiewer.rate_hw(best_student, 'Python', 10)
# cool_Rewiewer.rate_hw(best_student, 'Python', 7)
# cool_Rewiewer.rate_hw(best_student, 'Python', 9)

# print(best_student.grades)

# Задание 2
# cool_Lecturer = Lecturer('Someone', 'Someone-else')
# cool_Lecturer.courses_attached += ['Python']
# best_student.rate_lector(cool_Lecturer, 'Python', 9)
# best_student.rate_lector(cool_Lecturer, 'Python', 8)
# best_student.rate_lector(cool_Lecturer, 'Python', 7)
# best_student.rate_lector(cool_Lecturer, 'Python', 9)

# print(cool_Lecturer.grades)

# задание 3
# best_student.finished_courses += ['Some course #1', 'Some course #2']
# print(cool_Rewiewer)
# print(cool_Lecturer)
# print(best_student)
# print(cool_Lecturer < best_student)

# задание 4
student_1 = Student('Maksim', 'Petrov', 35)
student_1.courses_in_progress += ['Python', 'GIT']
student_1.finished_courses += ['Some course']
student_2 = Student('Evgeniy', 'Fedorov', 25)
student_2.courses_in_progress += ['Python', 'GIT']
student_2.finished_courses += ['Some course']

rewiewer_1 = Rewiewer('Aleksand', 'Ivanov')
rewiewer_1.courses_attached += ['Python']
rewiewer_2 = Rewiewer('Maria', 'Tesnova')
rewiewer_2.courses_attached += ['GIT']
rewiewer_1.rate_hw(student_1, 'Python', 9)
rewiewer_1.rate_hw(student_1, 'Python', 8)
rewiewer_1.rate_hw(student_2, 'Python', 10)
rewiewer_1.rate_hw(student_2, 'Python', 9)
rewiewer_2.rate_hw(student_1, 'GIT', 8)
rewiewer_2.rate_hw(student_1, 'GIT', 10)
rewiewer_2.rate_hw(student_2, 'GIT', 9)
rewiewer_2.rate_hw(student_2, 'GIT', 10)

lecturer_1 = Lecturer('Aleksey', 'Belov')
lecturer_1.courses_attached += ['Python']
lecturer_2 = Lecturer('Elena', 'Shatova')
lecturer_2.courses_attached += ['GIT']

student_1.rate_lector(lecturer_1, 'Python', 9)
student_2.rate_lector(lecturer_1, 'Python', 8)
student_1.rate_lector(lecturer_2, 'GIT', 10)
student_2.rate_lector(lecturer_2, 'GIT', 9)

print(student_1.grades)
print(student_2.grades)
print(lecturer_1.grades)
print(lecturer_2.grades)
print(rewiewer_1)
print(rewiewer_2)
print(student_1)
print(student_2)
print(lecturer_1 < student_1)
print(lecturer_2 < student_2)

# задание 4
student_list = [student_1, student_2]

def student_average_rating(student_list, course):
    a = []
    b = 0
    for name in student_list:
        if course in name.grades:
            for i in name.grades[course]:
                b += i
                a.append(i)
    av_rating = b / len(a)
    print(f'средняя оценка: {av_rating} за домашние задания по всем студентам в рамках курса "{course}"')
student_average_rating(student_list, 'GIT')
student_average_rating(student_list, 'Python')

lecturer_list = [lecturer_1, lecturer_2]

def lecturer_average_rating(lecturer_list, course):
    a = []
    b = 0
    for name in lecturer_list:
        if course in name.grades:
            for i in name.grades[course]:
                b += i
                a.append(i)
    av_rating = b / len(a)
    print(f'средняя оценка: {av_rating} за лекции всех лекторов в рамках курса "{course}"')
lecturer_average_rating(lecturer_list, 'Python')
lecturer_average_rating(lecturer_list, 'GIT')


import os
import numpy as np
import h5py
import scipy.io as scio

def preprocess(x, mean, std):
    mean, std = np.array(mean), np.array(std)
    return (x - mean.reshape(3, 1, 1)) / (std.reshape(3, 1, 1) + 1e-5)

def load_data(path, type='ucm'):
    print('Loading', type)
    if type == 'ucm' or type == 'rsicd':
        return load_ucm(path)
    elif type == 'flickr25k':
        return load_flickr25k(path)
    else:
        return load_nus_wide(path)

def load_flickr25k(path):
    data_file = h5py.File(path)
    images = data_file['images'][:]
    tags = data_file['YAll'][:]
    labels = data_file['LAll'][:]
    data_file.close()
    return images, tags, labels

def load_nus_wide(path_dir):
    image_file = h5py.File(os.path.join(path_dir, 'nus-wide-tc21-iall.mat'))
    images = image_file['IAll'][:]
    image_file.close()

    tag_data = scio.loadmat(os.path.join(path_dir, 'nus-wide-tc21-yall.mat'))
    tags = tag_data['YAll']

    label_data = scio.loadmat(os.path.join(path_dir, 'nus-wide-tc21-lall.mat'))
    labels = label_data['LAll']

    return images, tags, labels

def data_enhance(images, tags, labels, c=0.5):
    num = images.shape[0]
    ind1 = np.random.permutation(num // 2)
    ind2 = np.random.permutation(num // 2)
    inhanced_imgs = c * images[ind1] + (1 - c) * images[ind2]
    inhanced_tags = c * tags[ind1] + (1 - c) * tags[ind2]
    inhanced_labels = c * labels[ind1] + (1 - c) * labels[ind2]
    new_imgs = np.concatenate((images, inhanced_imgs))
    new_tags = np.concatenate((tags, inhanced_tags))
    new_labels = np.concatenate((labels, inhanced_labels))
    return new_imgs, new_tags, new_labels

def load_ucm(path):
    with h5py.File(path, "r") as hf:
        np.random.seed(42)
        images = hf['images'][:]
        images = (images - images.mean()) / images.std()
        labels = hf['labels'][:]
        tags = hf['bow'][:]  # embeddings
        tags = (tags - tags.mean()) / tags.std()
        images = duplicate_data(images, 5)  # 5 times more captions than images
        labels = duplicate_data(labels, 5)  # 5 times more captions than labels
        perm = np.random.permutation(len(images))
        images = images[perm]
        labels = labels[perm]
        tags = tags[perm]
    return images, tags, labels

def duplicate_data(data, n):
    """
    Duplicates each value of 0-dim n times
        for n = 3: (1, 2, 3) -> (1, 1, 1, 2, 2, 2, 3, 3, 3)

    :param data: original data
    :param n: number of duplications
    :return:
    """
    new_data = np.zeros((data.shape[0] * n, data.shape[1]), dtype=data.dtype)
    idx = 0
    for d in data:
        for i in range(n):
            new_data[idx] = d
            idx += 1
    return new_data

def load_pretrain_model(path):
    return scio.loadmat(path)


#!/usr/bin/env python3

import os
import shutil

R = '\033[31m' # red
G = '\033[32m' # green
C = '\033[36m' # cyan
W = '\033[0m'  # white

title = input(G + '[+]' + C + ' Имя группы : ' + W)
image = input(G + '[+]' + C + ' Путь к картинке группы (Рекомендуемый размер : 300x300): ' + W)

img_name = image.split('/')[-1]
try:
    shutil.copyfile(image, 'template/whatsapp/images/{}'.format(img_name))
except Exception as e:
    print('\n' + R + '[-]' + C + ' Exception : ' + W + str(e))
    exit()

with open('template/whatsapp/index_temp.html', 'r') as index_temp:
    code = index_temp.read()
    code = code.replace('$TITLE$', title)
    code = code.replace('$IMAGE$', 'images/{}'.format(img_name))

with open('template/whatsapp/index.html', 'w') as new_index:
    new_index.write(code)


import os
from flask_sqlalchemy import SQLAlchemy

database_path = os.environ['DATABASE_URL']
db = SQLAlchemy()

# ------------------------------------------------ #
#  Database config.
# ------------------------------------------------ #

def setup_db(app, database_uri=database_path):
    app.config['SQLALCHEMY_DATABASE_URI'] = database_path
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = os.environ['SQLALCHEMY_TRACK_MODIFICATIONS']
    db.app = app
    db.init_app(app)
    db.create_all()

# ------------------------------------------------ #
#  Models.
# ------------------------------------------------ #

class Todo(db.Model):
  __tablename__ = 'todos'
  id = db.Column(db.Integer, primary_key=True)
  description = db.Column(db.String(), nullable=False)
  completed = db.Column(db.Boolean, nullable=False, default=False)
  list_id = db.Column(db.Integer, db.ForeignKey('todolists.id'), nullable=False)

  def __repr__(self):
      return f'<Todo {self.id} {self.description} - completed: {self.completed}>'

class TodoList(db.Model):
  __tablename__ = 'todolists'
  id = db.Column(db.Integer, primary_key=True)
  name = db.Column(db.String(), nullable = False)
  completed = db.Column(db.Boolean, nullable=False, default=False)
  todos = db.relationship('Todo', backref='list', lazy=True)

  def __repr__(self):
    return f'<TodoList {self.id} {self.name}>'


# Make a flappy bird game using pygame

import pygame
import random
import os

# Initialize pygame
pygame.init()

# Create the screen
screen = pygame.display.set_mode((800, 600))

directory = os.path.dirname(__file__) + "\\"

# Background
background = pygame.image.load(directory + 'background.png')
background = pygame.transform.scale(background, (800, 600))
backgroundX = 0

# Caption and Icon
pygame.display.set_caption("Flappy Bird")
icon = pygame.image.load(directory + 'logo.png')
pygame.display.set_icon(icon)

# Player
playerImg = pygame.image.load(directory + 'bird.png')
playerX = 400
playerY = 300
playerX_change = 0
playerY_change = 0
num_of_obstacles = 0
# Spawn Player
screen.blit(playerImg, (playerX, playerY))

def player(x, y):
    # Move the player up smoothly by a block
    if y < 0:
        y = 0
    elif y > 600:
        y = 600
    screen.blit(playerImg, (x, y))

# funtion to check if player is out of bounds
def checkBounds(x, y):
    if x < 0 or x > 800 or y < 0 or y > 600:
        return True
    return False

def spawnObstacle():
    # draw the obstacle pipe, randomly
    obstacle = pygame.image.load(directory + 'obstacle.png')
    obstacleX = random.randint(0, 800)
    obstacleY = random.randint(0, 600)
    screen.blit(obstacle, (obstacleX, obstacleY))

# Game Loop
running = True
while running:
    playerX_change = 0
    playerY_change = 0

    # RGB - Red, Green, Blue
    screen.fill((0, 0, 0))

    backgroundX -= 0.1
    if backgroundX < -800:
        backgroundX = 0
    screen.blit(background, (backgroundX, 0))
    screen.blit(background, (backgroundX + 800, 0))
    if num_of_obstacles < 10:
        spawnObstacle()
        num_of_obstacles += 1
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_SPACE:
                playerY_change = -10
                pygame.mixer.music.load(directory + 'jump.wav')
                pygame.mixer.music.play(0)

    playerX += playerX_change
    playerY += playerY_change
    print("X = " + str(playerX))
    print("Y = " + str(playerY))
    player(playerX, playerY)
    if checkBounds(playerX, playerY):
        pygame.mixer.music.load(directory + 'gameover.wav')
        pygame.mixer.music.play(0)
        running = False
        print("Game Over")
    pygame.display.update()


# 大马3包，中马2包，小马0.5包
# 100匹马100包，问各有多少马

bigHorse = 3
midHorse = 2
tinyHorse = 0.5
bags = 100
horses = 100

for bi in range(0, bags // bigHorse + 1):
    for mi in range(0, bags // midHorse + 1):
        for ti in range(0, int(bags // tinyHorse + 1)):
            if (bi * bigHorse + mi * midHorse + ti * tinyHorse == bags) and (bi + mi + ti == horses):
                print('There are', bi, 'Big Horses,', mi, 'Midium Horses and', ti, 'Tiny Horses.')


import string
import math

print(f"Cálculo - 2 raízes de uma equação de segundo grau")

valores = []

lista = list(string.ascii_uppercase)

for i in range(0, 3):
    num = int(input(f'Informe o valor de {lista[i]}: '))
    valores.append(num)

delta = pow((valores[1] * -1), 2) - 4 * valores[0] * valores[2]

print('{}'.format('=' * 30))
print(f'Valor de Delta: {delta}')

if delta < 0:
    print(
        f'Delta: {delta}\nDelta menor do que ZERO, a equação não possui raízes reais')
elif delta == 0:
    raiz = (valores[1] * -1) / (2 * valores[0])
    print(f'A equação tem apenas 1 raíz\nRaiz: {raiz}')
elif delta > 0:
    raiz1 = ((valores[1] * -1) + math.sqrt(delta)) / (2 * valores[0])
    raiz2 = ((valores[1] * -1) - math.sqrt(delta)) / (2 * valores[0])

    print(f'A equação tem 2 raízes\nRaiz 1: {raiz1:.1f}\nRaiz 2: {raiz2:.1f}')


from __future__ import division

import numpy as np

import chainer
from chainer import cuda
import chainer.functions as F
from chainercv.links.model.faster_rcnn.utils.loc2bbox import loc2bbox
from chainercv.utils import non_maximum_suppression
from chainercv.transforms.image.resize import resize

class MaskRCNN(chainer.Chain):
    def __init__(self, extractor, rpn, head, mean,
                 min_size=600, max_size=1000,
                 loc_normalize_mean=(0., 0., 0., 0.),
                 loc_normalize_std=(0.1, 0.1, 0.2, 0.2),
                 ):
        print("MaskRCNN initialization")
        super(MaskRCNN, self).__init__()
        with self.init_scope():
            self.extractor = extractor
            self.rpn = rpn
            self.head = head

        self.mean = mean
        self.min_size = min_size
        self.max_size = max_size
        self.loc_normalize_mean = loc_normalize_mean
        self.loc_normalize_std = loc_normalize_std
        self.use_preset('visualize')
    @property
    def n_class(self):
        return self.head.n_class

    def __call__(self, x, scale=1.):
        img_size = x.shape[2:]
        h = self.extractor(x) #VGG
        rpn_locs, rpn_scores, rois, roi_indices, anchor = \
            self.rpn(h, img_size, scale) #Region Proposal Network
        roi_cls_locs, roi_scores, masks = self.head(
            h, rois, roi_indices) #Heads
        return roi_cls_locs, roi_scores, rois, roi_indices, masks

    def use_preset(self, preset):
        if preset == 'visualize':
            self.nms_thresh = 0.3
            self.score_thresh = 0.7
        elif preset == 'evaluate':
            self.nms_thresh = 0.3
            self.score_thresh = 0.05
        else:
            raise ValueError('preset must be visualize or evaluate')

    def prepare(self, img):
        _, H, W = img.shape
        scale = self.min_size / min(H, W)
        if scale * max(H, W) > self.max_size:
            scale = self.max_size / max(H, W)
        img = resize(img, (int(H * scale), int(W * scale)))
        img = (img - self.mean).astype(np.float32, copy=False)
        return img

    def _suppress(self, raw_cls_bbox, raw_cls_roi, raw_prob, raw_mask):
        bbox = list()
        roi = list()
        label = list()
        score = list()
        mask = list()
        for l in range(1, self.n_class):
            cls_bbox_l = raw_cls_bbox.reshape((-1, self.n_class, 4))[:, l, :]
            cls_roi_l = raw_cls_roi.reshape((-1, self.n_class, 4))[:, l, :]
            prob_l = raw_prob[:, l]
            lmask = prob_l > self.score_thresh
            cls_bbox_l = cls_bbox_l[lmask]
            cls_roi_l = cls_roi_l[lmask]
            prob_l = prob_l[lmask]
            mask_l = raw_mask[:,l]
            mask_l = mask_l[lmask]
            keep = non_maximum_suppression(cls_bbox_l, self.nms_thresh, prob_l)
            bbox.append(cls_bbox_l[keep])
            roi.append(cls_roi_l[keep])
            #labels are in [0, self.nclass - 2].
            label.append((l - 1) * np.ones((len(keep),)))
            score.append(prob_l[keep])
            mask.append(mask_l[keep])
        bbox = np.concatenate(bbox, axis=0).astype(np.float32)
        roi = np.concatenate(roi, axis=0).astype(np.float32)
        label = np.concatenate(label, axis=0).astype(np.float32)
        score = np.concatenate(score, axis=0).astype(np.float32)
        mask = np.concatenate(mask, axis=0).astype(np.float32)
        return bbox, roi,  label, score, mask

    def predict(self, imgs):
        prepared_imgs = list()
        sizes = list()
        print("predicting!")
        for img in imgs:
            size = img.shape[1:]
            img = self.prepare(img.astype(np.float32))
            prepared_imgs.append(img)
            sizes.append(size)
        bboxes = list()
        out_rois = list()
        labels = list()
        scores = list()
        masks = list()
        for img, size in zip(prepared_imgs, sizes):
            with chainer.using_config('train', False), \
                chainer.function.no_backprop_mode():
                img_var = chainer.Variable(self.xp.asarray(img[None]))
                scale = img_var.shape[3] / size[1]
                roi_cls_locs, roi_scores, rois, _,  roi_masks = self.__call__(img_var, scale=scale)

            #assuming batch size = 1
            roi_cls_loc = roi_cls_locs.data
            roi_score = roi_scores.data
            roi_mask = F.sigmoid(roi_masks).data
            roi = rois / scale
            mean = self.xp.tile(self.xp.asarray(self.loc_normalize_mean), self.n_class)
            std = self.xp.tile(self.xp.asarray(self.loc_normalize_std), self.n_class)
            roi_cls_loc = (roi_cls_loc * std + mean).astype(np.float32)
            roi_cls_loc = roi_cls_loc.reshape((-1, self.n_class, 4))
            roi = self.xp.broadcast_to(roi[:, None], roi_cls_loc.shape).reshape((-1, 4))
            cls_bbox = loc2bbox(roi, roi_cls_loc.reshape((-1, 4)))
            cls_bbox = cls_bbox.reshape((-1, self.n_class * 4))
            cls_roi = roi.reshape((-1, self.n_class * 4))
            #clip the bbox
            cls_bbox[:, 0::2] = self.xp.clip(cls_bbox[:, 0::2], 0, size[0])
            cls_bbox[:, 1::2] = self.xp.clip(cls_bbox[:, 1::2], 0, size[1])
            cls_roi[:, 0::2] = self.xp.clip(cls_roi[:, 0::2], 0, size[0])
            cls_roi[:, 1::2] = self.xp.clip(cls_roi[:, 1::2], 0, size[1])

            prob = F.softmax(roi_score).data

            raw_cls_bbox = cuda.to_cpu(cls_bbox)
            raw_cls_roi = cuda.to_cpu(cls_roi)
            raw_prob = cuda.to_cpu(prob)
            raw_mask = cuda.to_cpu(roi_mask)
            bbox, out_roi, label, score, mask = self._suppress(raw_cls_bbox, raw_cls_roi, raw_prob, raw_mask)
            bboxes.append(bbox)
            out_rois.append(out_roi)
            labels.append(label)
            scores.append(score)
            masks.append(mask)

        return bboxes, out_rois, labels, scores, masks


from Tkinter import *
from datetime import date, timedelta

from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2TkAgg
from gui.SymbolPlotter import SymbolPlotter
from utils.webdata import get_data_of_symbol
from analysis.basic import daily_prices_to_weekly_prices

class CanvasFrame(Frame):

    def __init__(self, parent):
        Frame.__init__(self, parent)

    def init_canvas(self, symbol):
        figure = self.__load_figure_of_symbol(symbol)
        canvas = FigureCanvasTkAgg(figure, self)
        self.canvas = canvas
        canvas.show()
        canvas.get_tk_widget().pack(fill=BOTH, expand=True)

        toolbar = NavigationToolbar2TkAgg(canvas, self)
        self.canvas_toolbar = toolbar
        toolbar.update()
        canvas._tkcanvas.pack(fill=BOTH, expand=True)

    def update_figure(self, symbol):
        figure = self.__load_figure_of_symbol(symbol)
        self.canvas.figure = figure
        self.canvas.draw()
        self.canvas_toolbar.update()

    def __load_figure_of_symbol(self, symbol):
        today = date.today()
        startdate = today - timedelta(days=3650)

        prices = get_data_of_symbol(symbol, startdate.isoformat(), today.isoformat(), fill_empty=False)
        # prices = daily_prices_to_weekly_prices(prices)
        plotter = SymbolPlotter(embed=True)
        figure = plotter.plot_single_symbol(prices, indicators={
            "volume" : None,
            # "bb" : None,
            # "macd" : None,
            # "sma" : {"windows": [5, 13]},
            # "ema" : {"windows": [5, 13]},
            # "rsi" : None,
            # "mfi" : None,
            # "cmf" : None,
            # "kdj" : {"windows": [9, 3, 3]},
            # "stoch" : {"windows": [9, 3, 3]},
            # "adx": {"window": 14},
            # "atr": {"window": 14},
            # "frac": {"draw_breakout": True},
            # "cci": {"window": 14},
            # "obv": None,
            # "adl": None,
            # "trix": {"windows": [15, 9]},
            "mafe": {"window": 5}
        })

        return figure


import os
import collections
import copy
import inspect
import logging
from datetime import date
from collections.abc import Mapping, Sequence

import cerberus
import cerberus.errors
import simtk.unit as unit
from simtk.openmm import app
import openmmtools as mmtools
from openmoltools.utils import unwrap_py2

from .. import utils, restraints

logger = logging.getLogger(__name__)

# ==============================================================================
# GLOBAL VARIABLES
# ==============================================================================

_NONBONDED_METHODS_STR = ['NoCutoff', 'CutoffPeriodic', 'CutoffNonPeriodic', 'Ewald', 'PME']
_NONBONDED_METHODS_APP = [getattr(app, nm) for nm in _NONBONDED_METHODS_STR]
_CUTOFF_NONBONDED_METHODS_STR = _NONBONDED_METHODS_STR[1:]
_CUTOFF_NONBONDED_METHODS_APP = _NONBONDED_METHODS_APP[1:]

# ==============================================================================
# YANK CUSTOM VALIDATOR CLASS FOR CERBERUS
# ==============================================================================

class YANKCerberusValidator(cerberus.Validator):
    """
    Custom cerberus.Validator class for YANK extending the Validator to include all the methods needed
    by YANK
    """

    # ====================================================
    # DEFAULT SETTERS
    # ====================================================

    def _normalize_default_setter_no_parameters(self, document):
        return dict(parameters=list())

    def _normalize_default_setter_tip4pew_or_none(self, document):
        # Default setting is before coercing so we need to check both strings and app objects.
        if document['nonbonded_method'] in _CUTOFF_NONBONDED_METHODS_STR + _CUTOFF_NONBONDED_METHODS_APP:
            return 'tip4pew'
        return None

    def _normalize_default_setter_0_molar_or_none(self, document):
        # Default setting is before coercing so we need to check both strings and app objects.
        if document['nonbonded_method'] in _CUTOFF_NONBONDED_METHODS_STR + _CUTOFF_NONBONDED_METHODS_APP:
            return '0.0*molar'
        return None

    # ====================================================
    # DATA COERCION
    # ====================================================

    def _normalize_coerce_single_str_to_list(self, value):
        """Cast a single string to a list of string"""
        if isinstance(value, str):
            return [value]
        return value

    def _normalize_coerce_str_to_openmm_app(self, value):
        """Convert a string to the openmm.app object with that name."""
        if isinstance(value, str):
            return to_openmm_app_coercer(value)
        return value

    def _normalize_coerce_str_to_unit(self, value):
        """Convert a string to a Quantity without checking for compatible units."""
        if isinstance(value, str):
            return to_unit_coercer(compatible_units=None)(value)
        return value

    def _normalize_coerce_str_to_distance_unit(self, value):
        """Convert a string to a Quantity with units compatible with Angstrom."""
        if isinstance(value, str):
            return to_unit_coercer(unit.angstrom)(value)
        return value

    def _normalize_coerce_str_to_molar_unit(self, value):
        """Convert a string to a Quantity with units compatible with molar."""
        if isinstance(value, str):
            return to_unit_coercer(unit.molar)(value)
        return value

    # ====================================================
    # DATA VALIDATORS
    # ====================================================

    def _check_with_file_exists(self, field, filepath):
        """Assert that the file is in fact, a file!"""
        if not os.path.isfile(filepath):
            self._error(field, 'File path {} does not exist.'.format(filepath))

    def _check_with_directory_exists(self, field, directory_path):
        """Assert that the file is in fact, a file!"""
        if not os.path.isdir(directory_path):
            self._error(field, 'Directory {} does not exist.'.format(directory_path))

    def _check_with_is_peptide(self, field, filepath):
        """Input file is a peptide."""
        extension = os.path.splitext(filepath)[1]
        if extension != '.pdb':
            self._error(field, "Not a .pdb file")

    def _check_with_is_small_molecule(self, field, filepath):
        """Input file is a small molecule."""
        file_formats = frozenset(['mol2', 'sdf', 'smiles', 'csv'])
        extension = os.path.splitext(filepath)[1][1:]
        if extension not in file_formats:
            self._error(field, 'File is not one of {}'.format(file_formats))

    def _check_with_positive_int_list(self, field, value):
        for p in value:
            if not isinstance(p, int) or not p >= 0:
                self._error(field, "{} of must be a positive integer!".format(p))

    def _check_with_int_or_all_string(self, field, value):
        if value != 'all' and not isinstance(value, int):
            self._error(field, "{} must be an int or the string 'all'".format(value))

    def _check_with_supported_system_file_format(self, field, file_paths):
        """Ensure the input system files are supported."""
        # Obtain the extension of the system files.
        file_extensions = {os.path.splitext(file_path)[1][1:] for file_path in file_paths}

        # Find a match for the extensions.
        expected_extensions = [
            ('amber', {'inpcrd', 'prmtop'}),
            ('amber', {'rst7', 'prmtop'}),
            ('gromacs', {'gro', 'top'}),
            ('openmm', {'pdb', 'xml'}),
            ('charmm', {'pdb', 'psf'})
        ]
        file_extension_type = None
        for extension_type, valid_extensions in expected_extensions:
            if file_extensions == valid_extensions:
                file_extension_type = extension_type
                break

        # Verify we found a match.
        if file_extension_type is None:
            self._error(field, 'must have file extensions matching one of the '
                               'following types: {}'.format(expected_extensions))
            return

        logger.debug('Correctly recognized {}files ({}) as {} '
                     'files'.format(field, file_paths, file_extension_type))

    def _check_with_is_valid_nonbonded_method(self, field, nonbonded_method):
        """Ensure the given nonbonded method is valid."""
        if nonbonded_method not in _NONBONDED_METHODS_APP:
            self._error(field, f'must be one of {_NONBONDED_METHODS_STR}')

    def _check_with_mandatory_with_cutoff(self, field, value):
        """Ensure the value is specified if the nonbonded method has a cutoff.

        The validator assumes that the document has a 'nonbonded_method' field.
        """
        if (self.document['nonbonded_method'] in _CUTOFF_NONBONDED_METHODS_APP and
                value is None):
            self._error(field, f'must be specified with the following nonbonded methods {_CUTOFF_NONBONDED_METHODS_STR}')

    def _check_with_only_with_cutoff(self, field, value):
        """Ensure the value is different than None only if the nonbonded method has a cutoff.

        The validator assumes that the document has a 'nonbonded_method' field.
        """
        if value is not None and self.document['nonbonded_method'] not in _CUTOFF_NONBONDED_METHODS_APP:
            self._error(field, f'can be specified only with the following nonbonded methods {_CUTOFF_NONBONDED_METHODS_STR}')

    def _check_with_only_with_no_cutoff(self, field, value):
        """Ensure the value is different than None only if the nonbonded method has no cutoff.

        The validator assumes that the document has a 'nonbonded_method' field.
        """
        if value is not None and self.document['nonbonded_method'] != app.NoCutoff:
            self._error(field, 'can be specified only if nonbonded method is NoCutoff')

    def _check_with_specify_lambda_electrostatics_and_sterics(self, field, value):
        """Check that the keys of a dictionary contain both lambda_electrostatics and lambda_sterics."""
        if ((isinstance(value, dict) or isinstance(value, collections.OrderedDict)) and
                not ('lambda_sterics' in value and 'lambda_electrostatics' in value)):
            self._error(field, "Missing required keys lambda_sterics and/or lambda_electrostatics")

    def _check_with_math_expressions_variables_are_given(self, field, value):
        """Check that in the alchemical path math expressions and function variables are correctly configured."""
        if not ( (isinstance(value, dict) or isinstance(value, collections.OrderedDict)) ):
            return
        # Check that there is at least one non-string value in the
        # alchemical path that may correspond to the end state values
        # of the mathematical expressions.
        string_entries = []
        for parameter_name, parameter_values in value.items():
            if isinstance(parameter_values, str):
                string_entries.append(parameter_name)

        # Check that there is at least 1 non-string entry.
        if len(string_entries) == len(value):
            self._error(field, "Only mathematical expressions have been given with no values for their variables")

        # If there are indeed mathematical expressions, make sure the function variable is given.
        if len(string_entries) > 0:
            try:
                function_variable_name = self.root_document['trailblazer_options']['function_variable_name']
            except KeyError:
                self._error(field, ("Mathematical expressions were detected but no function "
                                    "variable name was given in the 'trailblazer_options' section"))
            else:
                # The function variable should have exactly two end states.
                if len(value[function_variable_name]) != 2:
                    self._error(field, f"Only the two end-point values of function variable '{function_variable_name}' should be given.")

    def _check_with_lambda_between_0_and_1(self, field, value):
        """Ensure keys which are lambda values are in fact between 0 and 1"""
        if "lambda_" not in field or not isinstance(value, list):
            return

        collected_bad_values = []
        for single_value in value:
            if not (isinstance(single_value, float) and 0 <= single_value <= 1.0):
                collected_bad_values.append(single_value)

        if len(collected_bad_values):
            err_msg = "Entries with a 'lambda_' must be a float in [0, 1]. Values {} are not."
            self._error(field, err_msg.format(collected_bad_values))

    def _check_with_defined_in_alchemical_path(self, field, value):
        """When a function variable name is given in trailblazer_options, check that it's defined in alchemical_path."""
        if value not in self.root_document['alchemical_path']:
            self._error(field, f"Function variable name '{value}' is not defined in 'alchemical_path'")

    def _check_with_is_restraint_constructor(self, field, constructor_description):
        self._check_subclass_constructor(field, call_restraint_constructor, constructor_description)

    def _check_with_is_mcmc_move_constructor(self, field, constructor_description):
        self._check_subclass_constructor(field, call_mcmc_move_constructor, constructor_description)

    def _check_with_is_sampler_constructor(self, field, constructor_description):
        # Check if the MCMCMove is defined (its validation is done in
        # is_mcmc_move_constructor). Don't modify original dictionary.
        constructor_description = copy.deepcopy(constructor_description)
        constructor_description.pop('mcmc_moves', None)
        # Validate sampler.
        self._check_subclass_constructor(field, call_sampler_constructor, constructor_description)

    # ====================================================
    # DATA TYPES
    # ====================================================

    def _validate_type_quantity(self, value):
        if isinstance(value, unit.Quantity):
            return True

    # ====================================================
    # INTERNAL USAGE
    # ====================================================

    def _check_subclass_constructor(self, field, call_constructor_func,
                                    constructor_description):
        """Utility function for logging constructor validation errors."""
        try:
            call_constructor_func(constructor_description)
        except RuntimeError as e:
            self._error(field, str(e))

# ==============================================================================
# STATIC VALIDATORS/COERCERS
# ==============================================================================

def to_openmm_app_coercer(input_string):
    """
    Converter function to be used with :func:`yank.utils.validate_parameters`.

    Parameters
    ----------
    input_string : str
        Method name of openmm.app to fetch

    Returns
    -------
    method : Method of openmm.app
        Returns openmm.app.{input_string}
    """
    return getattr(app, input_string)

def to_unit_coercer(compatible_units):
    """Function generator to test unit bearing strings with Cerberus."""
    def _to_unit_coercer(quantity_str):
        return utils.quantity_from_string(quantity_str, compatible_units)
    return _to_unit_coercer

def to_integer_or_infinity_coercer(value):
    """Coerce everything that is not infinity to an integer."""
    if value != float('inf'):
        value = int(value)
    return value

def to_none_int_or_checkpoint(value):
    """Coerce-ish a value that is None, int, or "checkpoint" """
    if value is None or value == "checkpoint":
        return value
    else:
        return int(value)

# ==============================================================================
# OBJECT CONSTRUCTORS PARSING
# ==============================================================================

def _check_type_keyword(constructor_description):
    """Raise RuntimeError if 'type' is not in the description"""
    subcls_name = constructor_description.get('type', None)
    if subcls_name is None:
        raise RuntimeError("'type' must be specified")
    elif not isinstance(subcls_name, str):
        raise RuntimeError("'type' must be a string")

def call_constructor(parent_cls, constructor_description, special_conversions=None,
                     convert_quantity_strings=False, **default_kwargs):
    """Convert the constructor representation into an object.

    Parameters
    ----------
    parent_cls : type
        Only objects that inherit from this class are created.
    constructor_description : dict
        Must contain the key 'type' with the name (string) of the subclass
        of ``parent_cls`` and all keyword arguments to pass to its ``__init__``
        method.
    special_conversions : dict, optional
        Special conversions to pass to ``yank.utils.validate_parameters``.
    convert_quantity_strings : bool, optional
        If True, all the string constructor values are attempted to be
        converted to quantities before creating the object (default is
        False).
    **default_kwargs
        Overwrite the default value of the constructor keyword arguments.

    Returns
    -------
    obj: parent_cls
        The new instance of the object.

    Raises
    ------
    RuntimeError
        If an error occurred parsing the constructor description of
        while creating the object.

    See Also
    --------
    yank.utils.validate_parameters

    """
    if special_conversions is None:
        special_conversions = {}
    _check_type_keyword(constructor_description)

    # Check Retrieve subclass from 'type' keyword.
    constructor_description = copy.deepcopy(constructor_description)
    subcls_name = constructor_description.pop('type', None)
    try:
        subcls = mmtools.utils.find_subclass(parent_cls, subcls_name)
    except ValueError as e:
        raise RuntimeError(str(e))

    # Read the keyword arguments of the constructor.
    constructor_kwargs = utils.get_keyword_args(subcls.__init__, try_mro_from_class=subcls)

    # Overwrite eventual keyword arguments.
    for k, v in default_kwargs.items():
        if k in constructor_kwargs and k not in constructor_description:
            constructor_description[k] = v

    # Validate init keyword arguments.
    try:
        constructor_kwargs = utils.validate_parameters(
            parameters=constructor_description, template_parameters=constructor_kwargs,
            check_unknown=True, process_units_str=True, float_to_int=True,
            special_conversions=special_conversions
        )
    except (TypeError, ValueError) as e:
        raise RuntimeError('Validation of constructor failed with: {}'.format(str(e)))

    # Convert all string quantities if required.
    if convert_quantity_strings:
        for key, value in constructor_kwargs.items():
            try:
                quantity = utils.quantity_from_string(value)
            except:
                pass
            else:
                constructor_kwargs[key] = quantity

    # Attempt to instantiate object.
    try:
        obj = subcls(**constructor_kwargs)
    except Exception as e:
        raise RuntimeError('Attempt to initialize failed with: {}'.format(str(e)))

    return obj

def call_restraint_constructor(constructor_description):
    return call_constructor(restraints.ReceptorLigandRestraint, constructor_description,
                            convert_quantity_strings=True)

def call_mcmc_move_constructor(constructor_description, **default_kwargs):
    """Create an MCMCMove from a dict representation of the constructor parameters.

    Parameters
    ----------
    constructor_description : dict
        A dict containing the keyword ``'type'`` to indicate the class
        of the MCMCMove and all the keyword arguments to pass to its
        constructor. If ``'type'`` is ``'SequenceMove'``, the ``'move_list'``
        keyword should be a list of dict representations of the single
        moves.
    **default_kwargs
        Overwrite the the default keyword argument of the MCMCMove constructor
        or, if a ``SequenceMove`` is instantiated, the constructors of the move
        list.

    Returns
    -------
    mcmc_move : openmmtools.mcmc.MCMCMove
        The intantiation of the ``MCMCMove`` object.
    """
    # Check if this is a sequence of moves and we need to unnest the constructors.
    # If we end up with other nested constructors, we'll probably need a general strategy.
    _check_type_keyword(constructor_description)
    is_sequence_move = constructor_description['type'] == 'SequenceMove'
    if is_sequence_move and 'move_list' not in constructor_description:
        raise RuntimeError('A SequenceMove must specify a "move_list" keyword '
                           'containing a list of MCMCMoves')
    # Prepare nested MCMCMoves.
    if is_sequence_move:
        mcmc_moves_constructors = constructor_description['move_list']
    else:
        mcmc_moves_constructors = [constructor_description]
    # Call constructor of all moves.
    moves = []
    for mcmc_move_constructor in mcmc_moves_constructors:
        moves.append(call_constructor(mmtools.mcmc.MCMCMove, mcmc_move_constructor,
                                      convert_quantity_strings=True, **default_kwargs))
    if is_sequence_move:
        return mmtools.mcmc.SequenceMove(move_list=moves)
    return moves[0]

def call_sampler_constructor(constructor_description):
    special_conversions = {'number_of_iterations': to_integer_or_infinity_coercer,
                           'online_analysis_interval': to_none_int_or_checkpoint}
    return call_constructor(mmtools.multistate.MultiStateSampler, constructor_description,
                            special_conversions=special_conversions)

# ==============================================================================
# AUTOMATIC SCHEMA GENERATION
# ==============================================================================

def generate_unknown_type_validator(a_type):
    """
    Helper function to :func:`type_to_cerberus_map` to convert unknown types into a validator

    Parameters
    ----------
    a_type : type
        Any valid type, although this works for any type, its preferred to let :func:`type_to_cerberus_map`
        to invoke this method

    Returns
    -------
    validator_map : dict
        Map dictionary of form ``{'validator': <function>}`` where ``'validator'`` is literal and
        ``<function>`` is a validator function of type checker.
    """
    def nonstandard_type_validator(field, value, error):
        if type(value) is not a_type:
            error(field, 'Must be of type {}'.format(mmtools.utils.typename(a_type)))
    validator_dict = {'validator': nonstandard_type_validator}
    return validator_dict

def type_to_cerberus_map(a_type):
    """
    Convert a provided type to a valid Cerberus internal type dict.

    Parameters
    ----------
    a_type : type
        Type you want the converter to built-in Cerberus strings

    Returns
    -------
    type_map : dict
        Type map dictionary of form ``{'type': 'TYPE'}`` where ``'type'`` is literal and ``TYPE`` is the mapped
        type in Cerberus

        OR

         Map dictionary of form ``{'validator': <function>}`` where ``'validator'`` is literal and
        ``<function>`` is a validator function of type checker for non-standard

    Raises
    ------
    YANKCerberusUtilsError
        When the type has no known map
    """
    known_types = {bool: 'boolean',
                   bytes: 'binary',
                   bytearray: 'binary',
                   date: 'date',
                   Mapping: 'dict',
                   dict: 'dict',
                   float: 'float',
                   int: 'integer',
                   tuple: 'list',
                   list: 'list',
                   Sequence: 'list',
                   str: 'string',
                   set: 'set'}
    try:
        type_map = {'type': known_types[a_type]}
    except KeyError:
        type_map = generate_unknown_type_validator(a_type)
    return type_map

def generate_signature_schema(func, update_keys=None, exclude_keys=frozenset()):
    """Generate a dictionary to test function signatures with Cerberus' Schema.

    Parameters
    ----------
    func : function
        The function used to build the schema.
    update_keys : dict
        Keys in here have priority over automatic generation. It can be
        used to make an argument mandatory, or to use a specific validator.
    exclude_keys : list-like
        Keys in here are ignored and not included in the schema.

    Returns
    -------
    func_schema : dict
        The dictionary to be used as Cerberus Validator schema. Contains all keyword
        variables in the function signature as optional argument with
        the default type as validator. Unit bearing strings are converted.
        Argument with default None are always accepted. Camel case
        parameters in the function are converted to underscore style.

    Examples
    --------
    >>> from cerberus import Validator
    >>> def f(a, b, camelCase=True, none=None, quantity=3.0*unit.angstroms):
    ...     pass
    >>> f_dict = generate_signature_schema(f, exclude_keys=['quantity'])
    >>> print(isinstance(f_dict, dict))
    True
    >>> f_validator = Validator(generate_signature_schema(f))
    >>> f_validator.validated({'quantity': '1.0*nanometer'})
    {'quantity': Quantity(value=1.0, unit=nanometer)}

    """
    if update_keys is None:
        update_keys = {}

    func_schema = {}
    arg_spec = inspect.getfullargspec(unwrap_py2(func))
    args = arg_spec.args
    defaults = arg_spec.defaults

    # Check keys that must be excluded from first pass
    exclude_keys = set(exclude_keys)
    exclude_keys.update(update_keys)

    # Transform camelCase to underscore
    args = [utils.camelcase_to_underscore(arg) for arg in args]

    # Build schema
    optional_validator = {'required': False}  # Keys are always optional for this type
    for arg, default_value in zip(args[-len(defaults):], defaults):
        if arg not in exclude_keys:  # User defined keys are added later
            if default_value is None:  # None defaults are always accepted, and considered nullable
                validator = {'nullable': True}
            elif isinstance(default_value, unit.Quantity):  # Convert unit strings
                validator = {'coerce': to_unit_coercer(default_value.unit)}
            else:
                validator = type_to_cerberus_map(type(default_value))
            # Add the argument to the existing schema as a keyword
            # To the new keyword, add the optional flag and the "validator" flag
            # of either 'validator' or 'type' depending on how it was processed
            func_schema = {**func_schema, **{arg: {**optional_validator, **validator}}}

    # Add special user keys
    func_schema.update(update_keys)

    return func_schema


import re
from load_constants import teams, categories, years
from load_data import get_full_roster
from converters import convert_category
from utils import is_int

# parses any tweet to find players and teams
def tweet_parser(tweet):
    #parse a tweet and return a tuple of arrays of players and teams
    punctuation = re.compile("[.!?\\-@':,\n#]")
    tweet = punctuation.sub('', tweet)
    tweet = tweet.replace("#", "")
    tweet = tweet.replace("'", "")
    words = tweet.split(" ")
    p_players = []
    p_teams = []
    roster_df = get_full_roster(['player_name', 'first_name', 'last_name'])
    player_names, first, last = (roster_df['player_name'].str.lower().to_numpy(),
                                roster_df['first_name'].str.lower().to_numpy(),
                                roster_df['last_name'].str.lower().to_numpy())
    for i in range(len(words)):
        if i < len(words)-1:
            p_name = words[i] + " " + words[i+1]
        else:
            p_name = None
        p_team = words[i]
        if p_team.lower() in teams.values and p_team not in p_teams:
            p_teams.append(p_team)
        elif p_name is not None and p_name.lower() in player_names and p_name not in p_players:
            p_players.append(p_name)
    return (p_players, p_teams)

# parses a request tweet for players, teams, years and categorys
def mention_parser(tweet):
    print(tweet)
    at_tag = re.compile("@([a-zA-Z0-9]|[_])*", re.IGNORECASE)
    punctuation = re.compile("[!?\\-@:\n]")
    if tweet[0] == '.':
        tweet = tweet[1:]
    #parse a more specific and targeted tweet and return a player/team, year, category
    category = []
    player = []
    team = []
    year = []
    positions = []
    roster_df = get_full_roster(['player_name', 'first_name', 'last_name'])
    players, first, last = (roster_df['player_name'].str.lower().to_numpy(),
                                roster_df['first_name'].str.lower().to_numpy(),
                                roster_df['last_name'].str.lower().to_numpy())
    tweet = at_tag.sub('', tweet)
    tweet = punctuation.sub('', tweet)
    tweet = tweet.replace("'", "")
    while len(tweet) > 0 and tweet[0] == ' ':
        tweet = tweet[1:]
    words = tweet.split(",")
    for w in words:
        w = w.lower()
        if "allowed" in w:
          w = w.replace("allowed", "")
          defense = True
        else:
          defense = False
        if w == "":
            continue
        if "(" in w and ")" in w:
            p_pos = w.split("(")
            pos = p_pos[1].replace(")", "")
            w = w.replace("(" + pos + ")", "")
        else:
            pos = None
        while w[0] == ' ':
            w = w[1:]
        while w[-1] == ' ' or w[-1]==".":
            w = w[:-1]
        c = "default"
        if w[-1] == 's':
            c = w[:-1]
        cat1 = convert_category(w)
        cat2 = convert_category(c)
        if cat1 in categories and cat1 not in category:
            if defense:
              category.append(cat1+"_allowed")
            else:
              category.append(cat1)
        elif cat2 in categories and cat2 not in category:
            if defense:
              category.append(cat2+"_allowed")
            else:
              category.append(cat2)
        elif is_int(w) and w in years and int(w) not in year:
            year.append(int(w))
        elif w in teams.values and w not in team:
            team.append(w)
        elif w in players and w not in player:
            player.append(w)
            positions.append(pos)
        elif w in players and w in player and pos != None and pos not in positions:
            player.append(w)
            positions.append(pos)
        else:
            print("ERROR: Invalid parameter, skipping")
            continue
    if player == []:
        player = [None]
        positions=[None]
    if team == []:
        team = [None]
    if category == []:
        category = [None]
    if year == []:
        year = [None]
    return (player, team, year, category, positions)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2019-12-02 09:29:18
# @Author  : Guosheng Kang (guoshengkang@gmail.com)
# @Link    : https://guoshengkang.github.io
# @Version : $Id$

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

p_dnn_hidden_units = [2, 4, 8, 16, 32, 64, 128]
p_attention_factor = [2, 4, 8, 16, 32, 64, 128]
p_cin_layer_size = [2, 4, 8, 16, 32, 64, 128]
p_l2_reg = [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0]
p_dropout = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6]

df_loss = pd.read_csv('../result/Logloss_impact_of_parameters_1129.csv', index_col=0)
df_auc = pd.read_csv('../result/AUC_impact_of_parameters_1129.csv', index_col=0)

fig=plt.figure(figsize=(13, 6))

# Hidden Units
ax1=fig.add_subplot(2,5,1)
ax1.plot(range(7),df_loss.loc["dnn_hidden_units",],"bv-")
plt.xlabel("DNN Hidden Units\n(a)")
plt.ylabel("Logloss")
plt.xlim((0,6))
plt.xticks(range(7),["2","4","8","16","32","64","128"])
plt.grid(linestyle='-.')

ax2=fig.add_subplot(2,5,6)
ax2.plot(range(7),df_auc.loc["dnn_hidden_units",],"r^-")
plt.xlabel("DNN Hidden Units\n(f)")
plt.ylabel("AUC")
plt.xlim((0,6))
plt.xticks(range(7),["2","4","8","16","32","64","128"])
plt.grid(linestyle='-.')

# Cin_layer_size
ax1=fig.add_subplot(2,5,2)
ax1.plot(range(7),df_loss.loc["cin_layer_size",],"bv-")
plt.xlabel("CIN Layer Size\n(b)")
plt.ylabel("Logloss")
plt.xlim((0,6))
plt.xticks(range(7),["2","4","8","16","32","64","128"])
plt.grid(linestyle='-.')

ax2=fig.add_subplot(2,5,7)
ax2.plot(range(7),df_auc.loc["cin_layer_size",],"r^-")
plt.xlabel("CIN Layer Size\n(g)")
plt.ylabel("AUC")
plt.xlim((0,6))
plt.xticks(range(7),["2","4","8","16","32","64","128"])
plt.grid(linestyle='-.')

# Attention Factors
ax1=fig.add_subplot(2,5,3)
ax1.plot(range(7),df_loss.loc["attention_factor",],"bv-")
plt.xlabel("Attention Factor\n(c)")
plt.ylabel("Logloss")
plt.xlim((0,6))
plt.xticks(range(7),["2","4","8","16","32","64","128"])
plt.grid(linestyle='-.')

ax2=fig.add_subplot(2,5,8)
ax2.plot(range(7),df_auc.loc["attention_factor",],"r^-")
plt.xlabel("Attention Factor\n(h)")
plt.ylabel("AUC")
plt.xlim((0,6))
plt.xticks(range(7),["2","4","8","16","32","64","128"])
plt.grid(linestyle='-.')

# L2 Regularization
ax1=fig.add_subplot(2,5,4)
ax1.plot(range(7),df_loss.loc["l2_reg",],"bv-")
plt.xlabel("L2 Regularization\n(d)")
plt.ylabel("Logloss")
plt.xlim((0,6))
plt.xticks(range(7),["1e-6","1e-5","1e-4","1e-3","1e-2","1e-1","0"])
plt.grid(linestyle='-.')

ax2=fig.add_subplot(2,5,9)
ax2.plot(range(7),df_auc.loc["l2_reg",],"r^-")
plt.xlabel("L2 Regularization\n(i)")
plt.ylabel("AUC")
plt.xlim((0,6))
plt.xticks(range(7),["1e-6","1e-5","1e-4","1e-3","1e-2","1e-1","0"])
plt.grid(linestyle='-.')

# Dropout Rate
ax1=fig.add_subplot(2,5,5)
ax1.plot(range(7),df_loss.loc["dropout",],"bv-")
plt.xlabel("Dropout Rate\n(e)")
plt.ylabel("Logloss")
plt.xlim((0,6))
plt.xticks(range(7),["0","0.1","0.2","0.3","0.4","0.5","0.6"])
plt.grid(linestyle='-.')

ax2=fig.add_subplot(2,5,10)
ax2.plot(range(7),df_auc.loc["dropout",],"r^-")
plt.xlabel("Dropout Rate\n(j)")
plt.ylabel("AUC")
plt.xlim((0,6))
plt.xticks(range(7),["0","0.1","0.2","0.3","0.4","0.5","0.6"])
plt.grid(linestyle='-.')

plt.tight_layout() #设置默认的间距
plt.savefig('../result/impact_of_parameters_1129.png', dpi=200) #指定分辨率保存
plt.show()